DEVenv/devenver.py

#!/usr/bin/env python3

# DEVenver
# ------------------------------------------------------------------------------
# A simple python script to download portable applications and install them by
# unzipping them to a structured directory tree.

import urllib.request
import urllib.parse
import pathlib
import os
import tempfile
import hashlib
import shutil
import subprocess
import pprint
import argparse
import json
import importlib

from string import Template
from enum import Enum

# Internal
# ------------------------------------------------------------------------------
DOWNLOAD_CHUNK_SIZE = 1 * 1024 * 1024 # 1 megabyte
IS_WINDOWS          = os.name == "nt"

script_dir                 = os.path.dirname(os.path.abspath(__file__))
default_base_dir           = script_dir
default_base_downloads_dir = os.path.join(default_base_dir, 'Downloads')
default_base_install_dir   = os.path.join(default_base_dir, 'Install')

# Arguments
# ------------------------------------------------------------------------------
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('--downloads-dir',
                        help=f'Set the directory where downloaded files are cached (default: {default_base_downloads_dir})',
                        default=default_base_downloads_dir,
                        type=pathlib.Path)

arg_parser.add_argument('--install-dir',
                        help=f'Set the directory where downloaded files are installed (default: {default_base_install_dir})',
                        default=default_base_install_dir,
                        type=pathlib.Path)

arg_parser.add_argument('--manifest-file',
                        help=f'Python file that has a get_manifest() function returning a dictionary of applications to download & install (see manifest.py for starters)',
                        required=True,
                        type=pathlib.Path)

arg_parser.add_argument('--version',
                        action='version',
                        version='DEVenver v1')


args = arg_parser.parse_args()

base_downloads_dir = args.downloads_dir
base_install_dir   = args.install_dir

# ------------------------------------------------------------------------------
# This app list must always be installed, they provide the tools to install all
# other archives. Upon installation, we will collect the installation executable
# path and store them in global variables for the rest of the progam to use to
# unzip the files.
internal_app_list = []

internal_app_list.append({
    'label': '7zip',
    'manifests': [],
})

version = "920"
internal_app_list[-1]['manifests'].append({ # Download the bootstrap 7zip, this can be unzipped using shutils
    'download_checksum': '2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac',
    'download_url': f'https://www.7-zip.org/a/7za{version}.zip',
    'version': version,
    'executables': [
        {
            'path': '7za.exe',
            'symlink': [],
            'add_to_devenv_path': False,
            'checksum': 'c136b1467d669a725478a6110ebaaab3cb88a3d389dfa688e06173c066b76fcf'
        }
    ],
    'add_to_devenv_script': [],
})

version = "2201"
internal_app_list[-1]['manifests'].append({ # Download proper 7zip, extract this exe with the bootstrap 7zip
    'download_checksum': 'b055fee85472921575071464a97a79540e489c1c3a14b9bdfbdbab60e17f36e4',
    'download_url':      f'https://www.7-zip.org/a/7z{version}-x64.exe',
    'version': version,
    'executables': [
        {
            'path': '7z.exe',
            'symlink': [],
            'add_to_devenv_path': True,
            'checksum': '254cf6411d38903b2440819f7e0a847f0cfee7f8096cfad9e90fea62f42b0c23'
        }
    ],
    'add_to_devenv_script':  [],
})

# ------------------------------------------------------------------------------

version = "1.5.2"
internal_app_list.append({
    "label": "zstd",
    "manifests": [
        {
            "download_checksum": "68897cd037ee5e44c6d36b4dbbd04f1cc4202f9037415a3251951b953a257a09",
            "download_url": f"https://github.com/facebook/zstd/releases/download/v{version}/zstd-v{version}-win64.zip",
            "version": version,
            "executables": [
                {
                    "path": "zstd.exe",
                    "symlink": [],
                    "add_to_devenv_path": True,
                    "checksum": "f14e78c0651851a670f508561d2c5d647da0ba08e6b73231f2e7539812bae311",
                },
            ],
            "add_to_devenv_script": [],
        },
    ],
})

# ------------------------------------------------------------------------------

# These variables are set once they are downloaded dynamically and installed
# from the internal app listing!
zstd_exe           = ""
zip7_exe           = ""
zip7_bootstrap_exe = ""


# Functions
# ------------------------------------------------------------------------------
def print_header(title):
    line = f'> ' + title + ' ';
    print(line.ljust(100, '-'))

def lprint(*args, level=0, **kwargs):
    print(' ' + (' ' * 2 * level), *args, **kwargs)

def lexit(*args, level=0, **kwargs):
    print(' ' + (' ' * 2 * level), *args, **kwargs)
    exit()

def verify_file_sha256(file_path, checksum, label):
    if os.path.isfile(file_path) == False:
        exit(f'Cannot verify SHA256, path is not a file [path={file_path}]')

    result = False
    try:
        file = open(file_path, 'r+b')
        hasher = hashlib.sha256()
        hasher.update(file.read())
        derived_checksum = hasher.hexdigest()
        result = derived_checksum == checksum
        if result:
            lprint(f'- {label} SHA256 is good: {checksum}', level=1)
        else:
            lprint(f'- {label} SHA256 mismatch', level=1)
            lprint(f'  Expect: {checksum}', level=1)
            lprint(f'  Actual: {derived_checksum}', level=1)
    except PermissionError as exception:
        lprint(f"- {label} cannot verify SHA256 due to permission error, skipping", level=1)
        result = True
    else:
        file.close()

    return result

def download_file_at_url(url, download_path, download_checksum, label):
    # Check if file already downloaded and hashes match
    # --------------------------------------------------------------------------
    file_already_downloaded = False
    if os.path.isfile(download_path):
        lprint(f'- Cached archive found: {download_path}', level=1)
        file_already_downloaded = verify_file_sha256(download_path, download_checksum, 'Cached archive')
    else:
        lprint(f'- Download to disk: {download_path}', level=1)
        lprint(f'  URL: {url}', level=1)

    # Download the file from URL
    # --------------------------------------------------------------------------
    if file_already_downloaded == False:
        lprint('Initiating download request ...', level=2)
        with urllib.request.urlopen(url) as response:
            temp_file        = tempfile.mkstemp(text=False)
            temp_file_handle = temp_file[0]
            temp_file_path   = temp_file[1]
            temp_file_io     = os.fdopen(temp_file_handle, mode='w+b')

            download_failed = False
            try:
                line = ''
                total_download_size = int(response.getheader('Content-Length'))
                bytes_downloaded    = 0
                while chunk := response.read(DOWNLOAD_CHUNK_SIZE):
                    bytes_written      = temp_file_io.write(chunk)
                    bytes_downloaded  += bytes_written
                    percent_downloaded = int(bytes_downloaded / total_download_size * 100)

                    lprint(' ' * len(line), end='\r', level=2)
                    line = f'Downloading {percent_downloaded:.2f}% ({bytes_downloaded}/{total_download_size})'
                    lprint(line, end='\r', level=2)
            except Exception as exception:
                download_failed = True
                lprint(f'Download {label} from {url} failed, {exception}', level=2)
            finally:
                temp_file_io.close()
                print()

            if download_failed == True:
                os.remove(temp_file_path)
                exit()

            os.rename(temp_file_path, download_path)

    if file_already_downloaded == False:
        if verify_file_sha256(download_path, download_checksum, 'Downloaded archive') == False:
            exit()

class UnzipMethod(Enum):
    SHUTILS = 0
    ZIP7_BOOTSTRAP = 1
    DEFAULT = 2

def get_exe_install_dir(install_dir, label, version_label):
    result = pathlib.Path(install_dir, label.replace(' ', '_'), version_label)
    return result

def get_exe_install_path(install_dir, label, version_label, exe_rel_path):
    install_dir = get_exe_install_dir(install_dir, label, version_label)
    result      = pathlib.Path(install_dir, exe_rel_path)
    return result

def get_exe_symlink_dir(install_dir):
    result = pathlib.Path(install_dir, "Symlinks")
    return result

def download_and_install_archive(download_url,
                                 download_checksum,
                                 exe_list,
                                 version_label,
                                 label,
                                 unzip_method,
                                 download_dir,
                                 install_dir):

    exe_install_dir  = get_exe_install_dir(install_dir=install_dir,
                                           label=label,
                                           version_label=version_label)

    # Evaluate if we have already installed the requested archive
    # --------------------------------------------------------------------------
    exes_are_not_a_file = []
    exes_missing = []
    exes_present = []
    for exe_dict in exe_list:
        exe_path = get_exe_install_path(install_dir=install_dir,
                                        label=label,
                                        version_label=version_label,
                                        exe_rel_path=exe_dict['path'])

        if os.path.exists(exe_path) == True:
            if os.path.isfile(exe_path) == True:
                exes_present.append(exe_dict)
            else:
                exes_are_not_a_file.append(exe_dict)
        else:
            exes_missing.append(exe_dict)

    # Executables not install yet, verify, download and install if possible
    # --------------------------------------------------------------------------
    if len(exes_present) != len(exe_list):

        # Check if any of the manifest files are not files
        # ----------------------------------------------------------------------
        if len(exes_are_not_a_file) > 0: # Some item exists at the path but they are not files
            lprint(f'- {label} is installed but some of the expected executables are not a file!', level=1)
            for exe_dict in exes_are_not_a_file:
                lprint(f'    {exe_dict["path"]}', level=1)
            lprint(f'    Installation cannot proceed as unpacking would overwrite these paths', level=1)
            return

        # Check if any files are missing
        # ----------------------------------------------------------------------
        # Some executables are missing, some are available, installation will
        # trample over existing files, its not safe to unzip the archive as we may
        # overwrite config files or some other files that have been modified by the
        # program.
        #
        # Note that all files missing means we can assume that we haven't installed
        # yet ..
        if len(exes_missing) > 0 and len(exes_missing) != len(exe_list):
            lprint(f'- {label} is installed but some of the expected executables are missing from the installation!', level=1)
            for exe_dict in exes_are_not_a_file:
                lprint(f'    {exe_dict["path"]}', level=1)
            lprint(f'    Installation cannot proceed as unpacking could delete ', level=1)
            return

        assert(len(exes_missing) == len(exe_list))
        assert(len(exes_present) == 0)

        # Not installed yet, download and install
        # ----------------------------------------------------------------------
        # Determine the file name we are downloading from the URL
        download_url_parts = urllib.parse.urlparse(download_url)
        download_name      = pathlib.Path(urllib.parse.unquote(download_url_parts.path))

        # The path to move the temp file to after successful download, e.g.
        # download_dir  = C:/Dev/Downloads/Wezterm-windows-{version}.zip
        # download_name = Wezterm-windows-{version}.zip
        download_path = pathlib.Path(download_dir, download_name.name)

        # Download the archive at the URL
        download_file_at_url(download_url, download_path, download_checksum, label)

        # Install the archive by unpacking it
        # ----------------------------------------------------------------------
        if unzip_method == UnzipMethod.SHUTILS:
            lprint(f'- SHUtils unzip install {label} to: {exe_install_dir}', level=1)
            shutil.unpack_archive(download_path, exe_install_dir, 'zip')
        else:
            command = ''
            if unzip_method == UnzipMethod.ZIP7_BOOTSTRAP:
                command = f'"{zip7_bootstrap_exe}" x -bd "{download_path}" -o"{exe_install_dir}"'
                lprint(f'- 7z (bootstrap) unzip {label} to: {exe_install_dir}', level=1)
                lprint(f'  Command: {command}', level=1)
                subprocess.run(command)
            else:
                archive_path                    = download_path
                intermediate_zip_file_extracted = False

                # We could have a "app.zst" situation or an "app.tar.zst" situation
                #
                # "app.zst" only needs 1 extraction from the zstd tool
                # "app.tar.zst" needs 1 zstd extract and then 1 7zip extract
                #
                # When we have "app.tar.zst" we extract to the install folder, e.g.
                #
                # "app/1.0/app.tar"
                #
                # We call this an intermediate zip file, we will extract that file
                # with 7zip. After we're done, we will delete that _intermediate_
                # file to cleanup our install directory.
                if archive_path.suffix == '.zst':

                    archive_without_suffix = pathlib.Path(str(archive_path)[:-len(archive_path.suffix)]).name
                    next_archive_path = pathlib.Path(exe_install_dir, archive_without_suffix)

                    if os.path.exists(next_archive_path) == False:
                        command = f'"{zstd_exe}" --output-dir-flat "{exe_install_dir}" -d "{archive_path}"'
                        lprint(f'- zstd unzip {label} to: {exe_install_dir}', level=1)
                        lprint(f'  Command: {command}', level=1)

                        os.makedirs(exe_install_dir)
                        subprocess.run(command)

                    # Remove the extension from the file, we just extracted it
                    archive_path = next_archive_path

                    # If there's still a suffix after we removed the ".zst" we got
                    # an additional archive to unzip, e.g. "app.tar" remaining.
                    intermediate_zip_file_extracted = len(archive_path.suffix) > 0

                if len(archive_path.suffix) > 0:
                    command = f'"{zip7_exe}" x -aoa -spe -bso0 "{archive_path}" -o"{exe_install_dir}"'
                    command = command.replace('\\', '/')
                    lprint(f'- 7z unzip install {label} to: {exe_install_dir}', level=1)
                    lprint(f'  Command: {command}', level=1)
                    subprocess.run(command)

                if intermediate_zip_file_extracted:
                    lprint(f'- Detected intermediate zip file in install root, removing: {archive_path}', level=1)
                    os.remove(archive_path)

        # Remove duplicate root folder if detected
        # ----------------------------------------------------------------------
        # If after unpacking, there's only 1 directory in the install direction, we
        # assume that the zip contains a root folder. We will automatically merge
        # the root folder to the parent.
        has_files_in_install_dir = False
        dir_count                = 0
        dupe_root_folder_name    = ''
        with os.scandir(exe_install_dir) as scan_handle:
            for it in list(scan_handle):
                if it.is_file():
                    has_files_in_install_dir = True
                    break
                elif it.is_dir():
                    dupe_root_folder_name = it.name
                    dir_count            += 1
                if dir_count > 1:
                    break

        if dir_count == 1 and not has_files_in_install_dir:
            # There is only one folder after we unzipped, what happened here is
            # that the archive we unzipped had its contents within a root
            # folder. We will pull those files out because we already unzipped
            # into an isolated location for the application, e.g.
            #
            # Our install location C:/Dev/Install/7zip/920
            # After unzip C:/Dev/Install/7zip/920/7zip-920-x64
            #
            # We have an duplicate '7zip-920-x64' directory in our
            # installation. Move all the files in the duplicate directory up to
            # our '920' folder then remove the duplicate folder.
            dupe_root_folder_path = pathlib.Path(exe_install_dir, dupe_root_folder_name)
            lprint(f'- Detected duplicate root folder after unzip: {dupe_root_folder_path}', level=1)
            lprint(f'  Merging duplicate root folder to parent:    {exe_install_dir}', level=1)
            for file_name in os.listdir(dupe_root_folder_path):
                src = pathlib.Path(dupe_root_folder_path, file_name)
                dest = pathlib.Path(exe_install_dir, file_name)
                shutil.move(src, dest)

            os.rmdir(dupe_root_folder_path)

    # Verify the installation by checking the SHA256 of the executables
    # --------------------------------------------------------------------------
    exes_with_bad_hashes = []
    for exe_dict in exe_list:
        exe_rel_path = exe_dict['path']
        exe_path = get_exe_install_path(install_dir=install_dir,
                                        label=label,
                                        version_label=version_label,
                                        exe_rel_path=exe_rel_path)
        if os.path.isfile(exe_path) == False:
            lexit(f'- Installed {label} but could not find expected file for validating install: {exe_path}', level=1)

        if verify_file_sha256(file_path=exe_path, checksum=exe_dict['checksum'], label=exe_rel_path) == False:
            exes_with_bad_hashes.append(exe_dict)

    if len(exes_with_bad_hashes) > 0:
        lprint(f'- {label} is installed but executable SHA256 does not match!', level=1)
        lprint(f'  See hashes above, executable path(s):', level=1)
        for exe_dict in exes_with_bad_hashes:
            lprint(f'    {exe_dict["path"]}', level=1)
        lprint(f'  Something has modified the executable, this may be malicious or not!', level=1)
        lprint(f'  Manually uninstall the existing installation or amend the binary to be', level=1)
        lprint(f'  able to continue. Exiting.', level=1)
        exit()
    else:
        lprint(f'- {label} installed and valid: {exe_install_dir}', level=1)

    # Do the symlinks
    # --------------------------------------------------------------------------
    symlink_dir                   = get_exe_symlink_dir(install_dir)
    paths_to_add_to_devenv_script = set()
    for exe_dict in exe_list:
        exe_rel_path = exe_dict['path']
        exe_path = get_exe_install_path(install_dir=install_dir,
                                        label=label,
                                        version_label=version_label,
                                        exe_rel_path=exe_rel_path)

        for symlink_entry in exe_dict["symlink"]:
            symlink_dest = symlink_dir / symlink_entry
            symlink_src = exe_path
            skip_link = False;
            if os.path.exists(symlink_dest):
                # Windows uses hardlinks because symlinks require you to enable "developer" mode
                # Everyone else uses symlinks
                if (IS_WINDOWS and not os.path.isfile(symlink_dest)) or (not IS_WINDOWS and not os.path.islink(symlink_dest)):
                    lprint( "- Cannot create symlink! The destination file to create the symlink at.", level=1)
                    lprint( "  already exists and is *not* a link. We cannot remove this safely as we", level=1)
                    lprint( "  don't know what it is, exiting.", level=1)
                    lprint(f"  Symlink Source: {symlink_src}", level=1)
                    lexit (f"  Symlink Dest: {symlink_dest}", level=1)

                if os.path.samefile(symlink_dest, symlink_src):
                    skip_link = True
                else:
                    os.unlink(symlink_dest)

            if not skip_link:
                if IS_WINDOWS == True:
                    os.link(src=symlink_src, dst=symlink_dest)
                else:
                    os.symlink(src=symlink_src, dst=symlink_dest)

        # Collect paths to add to the devenv script
        # ----------------------------------------------------------------------
        if exe_dict['add_to_devenv_path'] == True:
            path = exe_path.parent.relative_to(install_dir)
            paths_to_add_to_devenv_script.add(path)

    global devenv_script_buffer
    for path in paths_to_add_to_devenv_script:
        if IS_WINDOWS:
            devenv_script_buffer += f"set PATH=%~dp0{path};%PATH%\n"
        else:
            devenv_script_buffer += f"PATH=$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" ) &> /dev/null && pwd ){path}\";%PATH%\n"

# Search the 2 dictionarries, 'first' and 'second' for the key. A matching key
# in 'first' taking precedence over the 'second' dictionary. If no key is
# found in either dictionaries then this function
# returns an empty string.
class ValidateAppListResult:
    def __init__(self):
        self.app_count = 0

def validate_app_list(app_list):
    result = ValidateAppListResult()
    manifest_rule_table = {
        'download_checksum':         'manifest must specify the SHA256 checksum for the downloaded file',
        'version':                   'manifest must specify the app version that is to be installed',
        'executables':               'manifest must specify an array of executable(s) for verifying installation',
        'download_url':              'manifest must specify the URL to download the app from',
        'add_to_devenv_script':      'manifest must specify an array of strings to inject into the portable development environment setup script',
    }

    executable_rule_table = {
        'path':               'executables must specify a path to a file from the installation to verify its checksum',
        'symlink':            'executables must specify an array of symlink names that will target the path',
        'add_to_devenv_path': 'executables must specify an boolean to indicate if the executable path should be added to the environment path',
        'checksum':           'executables must specify a string with the checksum of the executable',
    }

    for app in app_list:
        manifest_list = app['manifests']
        result.app_count += len(manifest_list)

        # Verify the label
        # ----------------------------------------------------------------------
        label = app.get('label', '')
        if 'label' not in app:
            exit('Label missing from application list, app must have a label specified, e.g. { "label": "App Name", "manifests": [] }')

        # Verify that the mandatory keys are in the manifest
        # ----------------------------------------------------------------------
        for manifest in manifest_list:
            for key in manifest_rule_table:
                value = manifest.get(key, "")

                if key.startswith("add_to_devenv"):
                    if not isinstance(value, list):
                        exit(f'{label} error: {key} in manifest must be an array to proceed\n{pprint.pformat(app)}')

                elif key == "executables":
                    for executable in value:
                        for executable_key in executable_rule_table:
                            executable_value = executable.get(executable_key, "")

                            if executable_key == "path":
                                if not isinstance(executable_value, str) or len(executable_value) == 0:
                                    exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')

                            elif executable_key == "symlink":
                                if not isinstance(executable_value, list):
                                    exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')

                            elif executable_key == "add_to_devenv_path":
                                if not isinstance(executable_value, bool):
                                    exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')

                            elif executable_key == "checksum":
                                if not isinstance(executable_value, str):
                                    exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')

                elif len(value) == 0:
                    exit(f'{label} error: required key "{key}" is missing/empty, {manifest_rule_table[key]}\n{pprint.pformat(app)}')

    return result

devenv_script_buffer = """@echo off

"""
def install_app_list(app_list, download_dir, install_dir):
    title = "Internal Apps" if app_list is internal_app_list else "User Apps"
    print_header(title)
    result = {}

    validate_app_list_result = validate_app_list(app_list)
    app_index = 0

    for app in app_list:
        manifest_list = app['manifests']
        for manifest in manifest_list:
            app_index += 1

            # Extract variables from manifest
            # ------------------------------------------------------------------
            label             = app["label"]
            download_checksum = manifest['download_checksum']
            version           = manifest["version"]
            download_url      = manifest["download_url"]
            exe_list          = manifest['executables']
            unzip_method      = UnzipMethod.DEFAULT

            # Bootstrapping code, when installing the internal app list, we will
            # assign the variables to point to our unarchiving tools.
            # ------------------------------------------------------------------
            if app_list is internal_app_list:
                global zip7_exe
                global zip7_bootstrap_exe
                global zstd_exe
                exe_path = get_exe_install_path(install_dir, label, version, manifest['executables'][0]['path'])
                if label == '7zip':
                    if version == '920':
                        unzip_method       = UnzipMethod.SHUTILS
                        zip7_bootstrap_exe = exe_path
                    else:
                        unzip_method = UnzipMethod.ZIP7_BOOTSTRAP
                        zip7_exe     = exe_path

                if label == 'zstd':
                    zstd_exe = exe_path


            # Download and install
            # ------------------------------------------------------------------
            lprint(f'[{app_index:03}/{validate_app_list_result.app_count:03}] Setup {label} v{version}', level=0)
            download_and_install_archive(download_url=download_url,
                                         download_checksum=download_checksum,
                                         exe_list=exe_list,
                                         version_label=version,
                                         label=label,
                                         unzip_method=unzip_method,
                                         download_dir=download_dir,
                                         install_dir=install_dir)

            # Post-installation
            # ------------------------------------------------------------------

            # Collate results into results
            if label not in result:
                result.update({label: []})

            for item in exe_list:
                app_install_dir = get_exe_install_dir(install_dir, label, version)
                app_exe_path    = get_exe_install_path(install_dir, label, version, item['path'])
                app_exe_dir     = pathlib.Path(app_exe_path).parent

                # Add executable into the result list
                result[label].append({
                    'version':     version,
                    'install_dir': app_install_dir,
                    'exe_path':    app_exe_path,
                })

            # Add the snippets verbatim specified in the manifest
            global devenv_script_buffer
            for line in manifest['add_to_devenv_script']:
                devenv_script_buffer += (line + '\n')

    if app_list is internal_app_list:
        if len(str(zip7_exe)) == 0 or len(str(zip7_bootstrap_exe)) == 0 or len(str(zstd_exe)) == 0:
            exit("Internal app list did not install 7zip bootstrap, 7zip or zstd, we are unable to install archives\n"
                 f" - zip7_bootstrap_exe: {zip7_bootstrap_exe}\n"
                 f" - zip7_exe: {zip7_exe}\n"
                 f" - zstd_exe: {zstd_exe}\n")

    return result

def run(user_app_list,
        download_dir=base_downloads_dir,
        install_dir=base_install_dir):
    """ Download and install the given user app list at the specified
    directories. The apps given must be archives that can be unpacked for
    installation (e.g. portable distributions).

    Parameters:
        user_app_list (list): A list of dictionaries that contain app and
        manifest information dictating what is to be installed via this
        function.
        download_dir (string): The path that intermediate downloaded files will
        be kept at.
        install_dir (string): The path that installed applications will be
        unpacked to

    Returns:
        result (list): A list of dictionaries containing the install locations
        of each app, e.g.
    """
    # Run
    # --------------------------------------------------------------------------
    # Create the starting directories and install the internal app list (e.g.
    # 7zip) which will be used to unzip-install the rest of the apps in the user
    # app list.
    #
    # To do this without dependencies, we first download an old version of 7zip,
    # version 9.20 which is distributed as a .zip file which Python can natively
    # unzip.
    #
    # We then use the old version of 7zip and download a newer version of 7zip
    # and extract it using the bootstrap-ed version. As of writing this, 7zip
    # does not release a portable distribution of itself yet, instead what we do
    # is download the installer and extract it ourselves using the bootstrap
    # 7zip.

    # Create the paths requested by the user
    os.makedirs(download_dir, exist_ok=True)
    os.makedirs(install_dir, exist_ok=True)
    os.makedirs(pathlib.Path(install_dir, "Symlinks"), exist_ok=True)

    for path in [download_dir, install_dir]:
        if not os.path.isdir(path):
            exit(f'Path "{path}" is not a directory, script can not proceed. Exiting.')

    # Validate all the manifests before starting
    internal_app_validate_result = validate_app_list(internal_app_list)
    user_app_validate_result     = validate_app_list(user_app_list)

    # Install apps
    internal_apps = install_app_list(app_list=internal_app_list,
                                     download_dir=download_dir,
                                     install_dir=install_dir)

    user_apps     = install_app_list(app_list=user_app_list,
                                     download_dir=download_dir,
                                     install_dir=install_dir)

    # Write the devenv script with environment variables
    global devenv_script_buffer
    devenv_script_buffer += "set PATH=%~dp0Symlinks;%PATH%\n"

    devenv_script_name = "devenv.bat" if IS_WINDOWS else "devenv.sh"
    devenv_script_path = pathlib.Path(install_dir, devenv_script_name)
    lprint(f"Writing script to augment the environment with installed applications: {devenv_script_path}")
    with open(devenv_script_path, 'w') as file:
        file.write(devenv_script_buffer)

    # Merge the install dictionaries, this dictionary contains
    # (app label) -> [array of installed versions]
    result = internal_apps
    for key, value in user_apps.items():
        if key not in result:
            result.update({key: value})
        else:
            result[key] += value

    return result

if __name__ == '__main__':
    run()