KB486777: DevOps Automation, Migrating objects with Packages With Python, MicroStrategy REST API and Github

Key Topics discussed in the demo:

• REST APIs for CICD workflows: Micro Strategy One has a great set of REST APIs that allow automating CICD workflows for moving objects between environments and storing them on GitHub. [0:00]

• Python script for creating and deploying packages: I presented a Python script in a Jupyter notebook that can find recently created objects, create packages and undo packages, deploy them to production, and push them to GitHub. [0:34]

• Rolling back changes with undo packages: I demonstrated how to download an undo package from GitHub and deploy it to production to reverse the changes made by a previous package. [6:10]

• Python service for MCE customers: I've described how MCE customers can have a nicer workflow with a Python service running on the server and a Strategy dashboard that displays and performs all the actions. [7:59]

Note - I've added color formatting in the code below (http://hilite.me/). Code copied from here might not work due to indentation issues and some problems with quotes (""). In that case, use attached file with sample code.
 

## Demo built by Robert Prochowicz - 2024-05-20
## using MSTR ONE March'24 and mstrio 11.3.12.101


# ## Import libraries and Functions

from mstrio.connection import Connection
from mstrio.object_management.migration import (
    bulk_full_migration,
    bulk_migrate_package,
    Migration,
    PackageConfig,
    PackageContentInfo,
    PackageSettings
)
from mstrio.types import ObjectTypes
from mstrio.project_objects.dossier import list_dossiers
from mstrio.project_objects.report import list_reports

from github import Github

import getpass, requests, base64
from datetime import datetime
from pathlib import Path
from os import listdir
from os.path import isfile, join
from pytz import timezone
import ipywidgets as widgets
from ipywidgets import Layout

# Create PackageConfig with information what object should be migrated and how.
# The options are of type Enum with all possible values listed.
package_settings = PackageSettings(
    PackageSettings.DefaultAction.USE_EXISTING,
    PackageSettings.UpdateSchema.RECAL_TABLE_LOGICAL_SIZE,
    PackageSettings.AclOnReplacingObjects.REPLACE,
    PackageSettings.AclOnNewObjects.KEEP_ACL_AS_SOURCE_OBJECT,
)

def pack_conf(mig_object, package_settings): # create PackageConfig
    package_content_info = PackageContentInfo(
        id=mig_object["id"],
        type=mig_object["type"],
        action=PackageContentInfo.Action.USE_EXISTING,
        include_dependents=True,
    )

    package_config = PackageConfig(
        PackageConfig.PackageUpdateType.PROJECT, package_settings, package_content_info
    )
    return package_config

def create_migration(mig_object, package_settings): # Perform many full migrations at once
    folder_path="" #can be defined here
    pconfig=pack_conf(mig_object, package_settings)
    now = datetime.now() # current date and time
    date_time = now.strftime("%Y-%m-%d--%H-%M-%S")
    filename=f"{date_time}-{pconfig.content[0].id}-{pconfig.content[0].type.name}.mpp"
    
    mig = Migration(
        save_path=filename,
        source_connection=source_conn,
        target_connection=target_conn,
        configuration=pconfig,
    )
    return mig, filename

def put_to_github(package_file): # this one is using requests library
    # Read the binary file
    with open(package_file, 'rb') as file:
        content = file.read()
    
    # Encode the content to base64
    encoded_content = base64.b64encode(content).decode('utf-8')
    
    # Prepare the request headers
    headers = {
        'Authorization': f'token {token}',
        'Accept': 'application/vnd.github.v3+json',
    }
    
    # Prepare the request payload
    payload = {
        'message': 'Add binary file via API',
        'content': encoded_content,
    }
    
    # Make the request to create or update the file
    response = requests.put(
        f'https://api.github.com/repos/{username}/{repo}/contents/{package_file}',
        json=payload,
        headers=headers
    )
    
    # Check the response
    if response.status_code == 201:
        print('File uploaded successfully.')
    else:
        print('Failed to upload file:', response.json())


def deploy_undo_package(UNDO_PACKAGE_PATH, target_conn):
    mig = Migration(
        save_path=UNDO_PACKAGE_PATH, target_connection=target_conn, custom_package_path=UNDO_PACKAGE_PATH
    ).migrate_package()
    return mig


# ## Establish MSTR Connections

# Define variables which can be later used in a script
PROJECT_NAME = "Strategy Tutorial"
SOURCE_BASE_URL = "https://env-297603.customer.cloud.Strategy.com/StrategyLibrary/api"  # usually ends with /StrategyLibrary/api
SOURCE_USERNAME = "mstr"
SOURCE_PASSWORD = getpass.getpass(prompt='Password Source ')
TARGET_BASE_URL = "https://env-334201.customer.cloud.Strategy.com/StrategyLibrary/api"  # usually ends with /StrategyLibrary/api
TARGET_USERNAME = "mstr"
TARGET_PASSWORD = getpass.getpass(prompt='Password Target ')

# Define a variable which can be later used iin a script
USERNAME = "mstr"  # user that is executing the migration

# Create connections to both source and target environments
source_conn = Connection(
    SOURCE_BASE_URL, SOURCE_USERNAME, SOURCE_PASSWORD, project_name=PROJECT_NAME, login_mode=1
)
target_conn = Connection(
    TARGET_BASE_URL, TARGET_USERNAME, TARGET_PASSWORD, project_name=PROJECT_NAME, login_mode=1
)


# ## Package creation

# #### Get all dashboards and reports

dashboards_list=list_dossiers(source_conn)
print(f"{len(dashboards_list)} dashboards downloaded")
reports_list=list_reports(source_conn)
print(f"{len(reports_list)} reports downloaded")

repdos_list=dashboards_list+reports_list


# #### Pick a date for Modification Date filter

date_from_w=widgets.Text(value="2024-01-01", description='Modif. Date:', disabled=False, indent=False, layout=Layout(width='20%'))
display(date_from_w)


# #### Object selection widget

def create_row(d):
    return [d.id, ' -- ', d.type, ' -- ', d.date_modified.strftime("%Y-%m-%d %H:%M:%S"), ' -- ' , d.name]

d_year, d_month, d_day = date_from_w.value.split('-')
dateFrom = datetime(int(d_year), int(d_month), int(d_day), 0, 0, 0, tzinfo=timezone('UTC'))

mig_objects_values = [create_row(dossier) for dossier in repdos_list if dossier.date_modified > dateFrom]
migobjects_widget=widgets.SelectMultiple(options=mig_objects_values, description='Select objects', disabled=False, layout=Layout(width="80%"), rows=10)
display(widgets.HBox(children=[migobjects_widget]))


# #### Full Migration: Create, download and push migration packages

selected_objects  = [{"id": a[0], "type": a[2]} for a in list(migobjects_widget.value)]
githubfiles=[]

for o in selected_objects:
    mig, mppfile=create_migration(o, package_settings)
    githubfiles.append(mppfile)
    githubfiles.append(mppfile[:-4]+"_undo.mpp")
    mig.perform_full_migration()

print(githubfiles)


# ## GITHUB

# pip install PyGithub
# https://medium.com/plumbersofdatascience/import-and-export-files-to-and-from-github-via-api-626efd7dd859
# PyGithub has some issues with uploading binary files, so I am using different methods for different tasks
# ordinary requests library - for pushing bineries to GitHub
# PyGithub - for listing files on Github and downloading files


# #### Push to Github

get_ipython().run_line_magic('run', 'migration_credentials.ipynb')
# Your GitHub username and repository
username = 'robert-prochowicz'
repo = 'mstr_migrations'
token = gt
token = "your_github_API_token"

for file in githubfiles:
    put_to_github(file)
    print(f"file {file} uploaded to Github\n")


# #### Download from Github

g = Github(gt)
grepo = g.get_repo(my_repo) # this is a PyGithub object

github_files=[file.name for file in grepo.get_contents("")]
migobjects_widget=widgets.SelectMultiple(options=github_files, description='Select objects', disabled=False, layout=Layout(width="80%"), rows=10)
display(widgets.HBox(children=[migobjects_widget]))

selected_objects  = [a for a in list(migobjects_widget.value)]
print()
# Get the files
for fg in selected_objects:
    file_content = grepo.get_contents(fg)

    # Download the file and save it on the disk
    with open(f'from_github\\{fg}', 'wb') as file:
        if file_content.encoding == 'base64':
            file.write(base64.b64decode(file_content.content))
            print(f"{fg} - base64")
        else:
            file.write(file_content.content.encode())
            print(f"{fg} - ___")


# #### Undo migration

# In case you want to roll-back implemented changes
for uo in selected_objects:
    deploy_undo_package(f'from_github\\{uo}', target_conn)


# #### Push to Github OLD

get_ipython().run_line_magic('run', 'migration_credentials.ipynb')
gtoken=gt
my_repo='robert-prochowicz/mstr_migrations'

g = Github(gtoken)
repo = g.get_repo(my_repo)

for file in githubfiles:
    with open(file, "rb") as f:
        bytes = f.read()
        b64_data = base64.b64encode(bytes)
    repo.create_file(file, 'upload image', b64_data)
    print(f"file {file} uploaded to Github")

with open("abc.mpp", "rb") as file:
    content = file.read()

# Encode the file content to base64
encoded_content = base64.b64encode(content)