736 lines
33 KiB
Python
736 lines
33 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
||
|
# DEVenver
|
||
|
# ------------------------------------------------------------------------------
|
||
|
# A simple python script to download portable applications and install them by
|
||
|
# unzipping them to a structured directory tree.
|
||
|
|
||
|
import urllib.request
|
||
|
import urllib.parse
|
||
|
import pathlib
|
||
|
import os
|
||
|
import tempfile
|
||
|
import hashlib
|
||
|
import shutil
|
||
|
import subprocess
|
||
|
import pprint
|
||
|
import argparse
|
||
|
import json
|
||
|
import importlib
|
||
|
|
||
|
from string import Template
|
||
|
from enum import Enum
|
||
|
|
||
|
# Internal
|
||
|
# ------------------------------------------------------------------------------
|
||
|
DOWNLOAD_CHUNK_SIZE = 1 * 1024 * 1024 # 1 megabyte
|
||
|
IS_WINDOWS = os.name == "nt"
|
||
|
|
||
|
script_dir = os.path.dirname(os.path.abspath(__file__))
|
||
|
default_base_dir = script_dir
|
||
|
default_base_downloads_dir = os.path.join(default_base_dir, 'Downloads')
|
||
|
default_base_install_dir = os.path.join(default_base_dir, 'Install')
|
||
|
|
||
|
# Arguments
|
||
|
# ------------------------------------------------------------------------------
|
||
|
arg_parser = argparse.ArgumentParser()
|
||
|
arg_parser.add_argument('--downloads-dir',
|
||
|
help=f'Set the directory where downloaded files are cached (default: {default_base_downloads_dir})',
|
||
|
default=default_base_downloads_dir,
|
||
|
type=pathlib.Path)
|
||
|
|
||
|
arg_parser.add_argument('--install-dir',
|
||
|
help=f'Set the directory where downloaded files are installed (default: {default_base_install_dir})',
|
||
|
default=default_base_install_dir,
|
||
|
type=pathlib.Path)
|
||
|
|
||
|
arg_parser.add_argument('--manifest-file',
|
||
|
help=f'Python file that has a get_manifest() function returning a dictionary of applications to download & install (see manifest.py for starters)',
|
||
|
required=True,
|
||
|
type=pathlib.Path)
|
||
|
|
||
|
arg_parser.add_argument('--version',
|
||
|
action='version',
|
||
|
version='DEVenver v1')
|
||
|
|
||
|
|
||
|
args = arg_parser.parse_args()
|
||
|
|
||
|
base_downloads_dir = args.downloads_dir
|
||
|
base_install_dir = args.install_dir
|
||
|
|
||
|
# ------------------------------------------------------------------------------
|
||
|
# This app list must always be installed, they provide the tools to install all
|
||
|
# other archives. Upon installation, we will collect the installation executable
|
||
|
# path and store them in global variables for the rest of the progam to use to
|
||
|
# unzip the files.
|
||
|
internal_app_list = []
|
||
|
|
||
|
internal_app_list.append({
|
||
|
'label': '7zip',
|
||
|
'manifests': [],
|
||
|
})
|
||
|
|
||
|
version = "920"
|
||
|
internal_app_list[-1]['manifests'].append({ # Download the bootstrap 7zip, this can be unzipped using shutils
|
||
|
'download_checksum': '2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac',
|
||
|
'download_url': f'https://www.7-zip.org/a/7za{version}.zip',
|
||
|
'version': version,
|
||
|
'executables': [
|
||
|
{
|
||
|
'path': '7za.exe',
|
||
|
'symlink': [],
|
||
|
'add_to_devenv_path': False,
|
||
|
'checksum': 'c136b1467d669a725478a6110ebaaab3cb88a3d389dfa688e06173c066b76fcf'
|
||
|
}
|
||
|
],
|
||
|
'add_to_devenv_script': [],
|
||
|
})
|
||
|
|
||
|
version = "2201"
|
||
|
internal_app_list[-1]['manifests'].append({ # Download proper 7zip, extract this exe with the bootstrap 7zip
|
||
|
'download_checksum': 'b055fee85472921575071464a97a79540e489c1c3a14b9bdfbdbab60e17f36e4',
|
||
|
'download_url': f'https://www.7-zip.org/a/7z{version}-x64.exe',
|
||
|
'version': version,
|
||
|
'executables': [
|
||
|
{
|
||
|
'path': '7z.exe',
|
||
|
'symlink': [],
|
||
|
'add_to_devenv_path': True,
|
||
|
'checksum': '254cf6411d38903b2440819f7e0a847f0cfee7f8096cfad9e90fea62f42b0c23'
|
||
|
}
|
||
|
],
|
||
|
'add_to_devenv_script': [],
|
||
|
})
|
||
|
|
||
|
# ------------------------------------------------------------------------------
|
||
|
|
||
|
version = "1.5.2"
|
||
|
internal_app_list.append({
|
||
|
"label": "zstd",
|
||
|
"manifests": [
|
||
|
{
|
||
|
"download_checksum": "68897cd037ee5e44c6d36b4dbbd04f1cc4202f9037415a3251951b953a257a09",
|
||
|
"download_url": f"https://github.com/facebook/zstd/releases/download/v{version}/zstd-v{version}-win64.zip",
|
||
|
"version": version,
|
||
|
"executables": [
|
||
|
{
|
||
|
"path": "zstd.exe",
|
||
|
"symlink": [],
|
||
|
"add_to_devenv_path": True,
|
||
|
"checksum": "f14e78c0651851a670f508561d2c5d647da0ba08e6b73231f2e7539812bae311",
|
||
|
},
|
||
|
],
|
||
|
"add_to_devenv_script": [],
|
||
|
},
|
||
|
],
|
||
|
})
|
||
|
|
||
|
# ------------------------------------------------------------------------------
|
||
|
|
||
|
# These variables are set once they are downloaded dynamically and installed
|
||
|
# from the internal app listing!
|
||
|
zstd_exe = ""
|
||
|
zip7_exe = ""
|
||
|
zip7_bootstrap_exe = ""
|
||
|
|
||
|
|
||
|
# Functions
|
||
|
# ------------------------------------------------------------------------------
|
||
|
def print_header(title):
|
||
|
line = f'> ' + title + ' ';
|
||
|
print(line.ljust(100, '-'))
|
||
|
|
||
|
def lprint(*args, level=0, **kwargs):
|
||
|
print(' ' + (' ' * 2 * level), *args, **kwargs)
|
||
|
|
||
|
def lexit(*args, level=0, **kwargs):
|
||
|
print(' ' + (' ' * 2 * level), *args, **kwargs)
|
||
|
exit()
|
||
|
|
||
|
def verify_file_sha256(file_path, checksum, label):
|
||
|
if os.path.isfile(file_path) == False:
|
||
|
exit(f'Cannot verify SHA256, path is not a file [path={file_path}]')
|
||
|
|
||
|
result = False
|
||
|
try:
|
||
|
file = open(file_path, 'r+b')
|
||
|
hasher = hashlib.sha256()
|
||
|
hasher.update(file.read())
|
||
|
derived_checksum = hasher.hexdigest()
|
||
|
result = derived_checksum == checksum
|
||
|
if result:
|
||
|
lprint(f'- {label} SHA256 is good: {checksum}', level=1)
|
||
|
else:
|
||
|
lprint(f'- {label} SHA256 mismatch', level=1)
|
||
|
lprint(f' Expect: {checksum}', level=1)
|
||
|
lprint(f' Actual: {derived_checksum}', level=1)
|
||
|
except PermissionError as exception:
|
||
|
lprint(f"- {label} cannot verify SHA256 due to permission error, skipping", level=1)
|
||
|
result = True
|
||
|
else:
|
||
|
file.close()
|
||
|
|
||
|
return result
|
||
|
|
||
|
def download_file_at_url(url, download_path, download_checksum, label):
|
||
|
# Check if file already downloaded and hashes match
|
||
|
# --------------------------------------------------------------------------
|
||
|
file_already_downloaded = False
|
||
|
if os.path.isfile(download_path):
|
||
|
lprint(f'- Cached archive found: {download_path}', level=1)
|
||
|
file_already_downloaded = verify_file_sha256(download_path, download_checksum, 'Cached archive')
|
||
|
else:
|
||
|
lprint(f'- Download to disk: {download_path}', level=1)
|
||
|
lprint(f' URL: {url}', level=1)
|
||
|
|
||
|
# Download the file from URL
|
||
|
# --------------------------------------------------------------------------
|
||
|
if file_already_downloaded == False:
|
||
|
lprint('Initiating download request ...', level=1)
|
||
|
with urllib.request.urlopen(url) as response:
|
||
|
temp_file = tempfile.mkstemp(text=False)
|
||
|
temp_file_handle = temp_file[0]
|
||
|
temp_file_path = temp_file[1]
|
||
|
temp_file_io = os.fdopen(temp_file_handle, mode='w+b')
|
||
|
|
||
|
download_failed = False
|
||
|
try:
|
||
|
line = ''
|
||
|
total_download_size = int(response.getheader('Content-Length'))
|
||
|
bytes_downloaded = 0
|
||
|
while chunk := response.read(DOWNLOAD_CHUNK_SIZE):
|
||
|
bytes_written = temp_file_io.write(chunk)
|
||
|
bytes_downloaded += bytes_written
|
||
|
percent_downloaded = int(bytes_downloaded / total_download_size * 100)
|
||
|
|
||
|
lprint(' ' * len(line), end='\r', level=1)
|
||
|
line = f'Downloading {percent_downloaded:.2f}% ({bytes_downloaded}/{total_download_size})'
|
||
|
lprint(line, end='\r', level=1)
|
||
|
except Exception as exception:
|
||
|
download_failed = True
|
||
|
lprint(f'Download {label} from {url} failed, {exception}', level=1)
|
||
|
finally:
|
||
|
temp_file_io.close()
|
||
|
print()
|
||
|
|
||
|
if download_failed == True:
|
||
|
os.remove(temp_file_path)
|
||
|
exit()
|
||
|
|
||
|
os.rename(temp_file_path, download_path)
|
||
|
|
||
|
if file_already_downloaded == False:
|
||
|
if verify_file_sha256(download_path, download_checksum, 'Downloaded archive') == False:
|
||
|
exit()
|
||
|
|
||
|
class UnzipMethod(Enum):
|
||
|
SHUTILS = 0
|
||
|
ZIP7_BOOTSTRAP = 1
|
||
|
DEFAULT = 2
|
||
|
|
||
|
def get_exe_install_dir(install_dir, label, version_label):
|
||
|
result = pathlib.Path(install_dir, label.replace(' ', '_'), version_label)
|
||
|
return result
|
||
|
|
||
|
def get_exe_install_path(install_dir, label, version_label, exe_rel_path):
|
||
|
install_dir = get_exe_install_dir(install_dir, label, version_label)
|
||
|
result = pathlib.Path(install_dir, exe_rel_path)
|
||
|
return result
|
||
|
|
||
|
def get_exe_symlink_dir(install_dir):
|
||
|
result = pathlib.Path(install_dir, "Symlinks")
|
||
|
return result
|
||
|
|
||
|
def download_and_install_archive(download_url,
|
||
|
download_checksum,
|
||
|
exe_list,
|
||
|
version_label,
|
||
|
label,
|
||
|
unzip_method,
|
||
|
download_dir,
|
||
|
install_dir):
|
||
|
|
||
|
exe_install_dir = get_exe_install_dir(install_dir=install_dir,
|
||
|
label=label,
|
||
|
version_label=version_label)
|
||
|
|
||
|
# Evaluate if we have already installed the requested archive
|
||
|
# --------------------------------------------------------------------------
|
||
|
exes_are_not_a_file = []
|
||
|
exes_missing = []
|
||
|
exes_present = []
|
||
|
for exe_dict in exe_list:
|
||
|
exe_path = get_exe_install_path(install_dir=install_dir,
|
||
|
label=label,
|
||
|
version_label=version_label,
|
||
|
exe_rel_path=exe_dict['path'])
|
||
|
|
||
|
if os.path.exists(exe_path) == True:
|
||
|
if os.path.isfile(exe_path) == True:
|
||
|
exes_present.append(exe_dict)
|
||
|
else:
|
||
|
exes_are_not_a_file.append(exe_dict)
|
||
|
else:
|
||
|
exes_missing.append(exe_dict)
|
||
|
|
||
|
# Executables not install yet, verify, download and install if possible
|
||
|
# --------------------------------------------------------------------------
|
||
|
if len(exes_present) != len(exe_list):
|
||
|
|
||
|
# Check if any of the manifest files are not files
|
||
|
# ----------------------------------------------------------------------
|
||
|
if len(exes_are_not_a_file) > 0: # Some item exists at the path but they are not files
|
||
|
lprint(f'- {label} is installed but some of the expected executables are not a file!', level=1)
|
||
|
for exe_dict in exes_are_not_a_file:
|
||
|
lprint(f' {exe_dict["path"]}', level=1)
|
||
|
lprint(f' Installation cannot proceed as unpacking would overwrite these paths', level=1)
|
||
|
return
|
||
|
|
||
|
# Check if any files are missing
|
||
|
# ----------------------------------------------------------------------
|
||
|
# Some executables are missing, some are available, installation will
|
||
|
# trample over existing files, its not safe to unzip the archive as we may
|
||
|
# overwrite config files or some other files that have been modified by the
|
||
|
# program.
|
||
|
#
|
||
|
# Note that all files missing means we can assume that we haven't installed
|
||
|
# yet ..
|
||
|
if len(exes_missing) > 0 and len(exes_missing) != len(exe_list):
|
||
|
lprint(f'- {label} is installed but some of the expected executables are missing from the installation!', level=1)
|
||
|
for exe_dict in exes_are_not_a_file:
|
||
|
lprint(f' {exe_dict["path"]}', level=1)
|
||
|
lprint(f' Installation cannot proceed as unpacking could delete ', level=1)
|
||
|
return
|
||
|
|
||
|
assert(len(exes_missing) == len(exe_list))
|
||
|
assert(len(exes_present) == 0)
|
||
|
|
||
|
# Not installed yet, download and install
|
||
|
# ----------------------------------------------------------------------
|
||
|
# Determine the file name we are downloading from the URL
|
||
|
download_url_parts = urllib.parse.urlparse(download_url)
|
||
|
download_name = pathlib.Path(urllib.parse.unquote(download_url_parts.path))
|
||
|
|
||
|
# The path to move the temp file to after successful download, e.g.
|
||
|
# download_dir = C:/Dev/Downloads/Wezterm-windows-{version}.zip
|
||
|
# download_name = Wezterm-windows-{version}.zip
|
||
|
download_path = pathlib.Path(download_dir, download_name.name)
|
||
|
|
||
|
# Download the archive at the URL
|
||
|
download_file_at_url(download_url, download_path, download_checksum, label)
|
||
|
|
||
|
# Install the archive by unpacking it
|
||
|
# ----------------------------------------------------------------------
|
||
|
if unzip_method == UnzipMethod.SHUTILS:
|
||
|
lprint(f'- SHUtils unzip install {label} to: {exe_install_dir}', level=1)
|
||
|
shutil.unpack_archive(download_path, exe_install_dir, 'zip')
|
||
|
else:
|
||
|
command = ''
|
||
|
if unzip_method == UnzipMethod.ZIP7_BOOTSTRAP:
|
||
|
command = f'"{zip7_bootstrap_exe}" x -bd "{download_path}" -o"{exe_install_dir}"'
|
||
|
lprint(f'- 7z (bootstrap) unzip {label} to: {exe_install_dir}', level=1)
|
||
|
lprint(f' Command: {command}', level=1)
|
||
|
subprocess.run(command)
|
||
|
else:
|
||
|
archive_path = download_path
|
||
|
intermediate_zip_file_extracted = False
|
||
|
|
||
|
# We could have a "app.zst" situation or an "app.tar.zst" situation
|
||
|
#
|
||
|
# "app.zst" only needs 1 extraction from the zstd tool
|
||
|
# "app.tar.zst" needs 1 zstd extract and then 1 7zip extract
|
||
|
#
|
||
|
# When we have "app.tar.zst" we extract to the install folder, e.g.
|
||
|
#
|
||
|
# "app/1.0/app.tar"
|
||
|
#
|
||
|
# We call this an intermediate zip file, we will extract that file
|
||
|
# with 7zip. After we're done, we will delete that _intermediate_
|
||
|
# file to cleanup our install directory.
|
||
|
if archive_path.suffix == '.zst':
|
||
|
|
||
|
archive_without_suffix = pathlib.Path(str(archive_path)[:-len(archive_path.suffix)]).name
|
||
|
next_archive_path = pathlib.Path(exe_install_dir, archive_without_suffix)
|
||
|
|
||
|
if os.path.exists(next_archive_path) == False:
|
||
|
command = f'"{zstd_exe}" --output-dir-flat "{exe_install_dir}" -d "{archive_path}"'
|
||
|
lprint(f'- zstd unzip {label} to: {exe_install_dir}', level=1)
|
||
|
lprint(f' Command: {command}', level=1)
|
||
|
|
||
|
os.makedirs(exe_install_dir)
|
||
|
subprocess.run(command)
|
||
|
|
||
|
# Remove the extension from the file, we just extracted it
|
||
|
archive_path = next_archive_path
|
||
|
|
||
|
# If there's still a suffix after we removed the ".zst" we got
|
||
|
# an additional archive to unzip, e.g. "app.tar" remaining.
|
||
|
intermediate_zip_file_extracted = len(archive_path.suffix) > 0
|
||
|
|
||
|
if len(archive_path.suffix) > 0:
|
||
|
command = f'"{zip7_exe}" x -aoa -spe -bso0 "{archive_path}" -o"{exe_install_dir}"'
|
||
|
command = command.replace('\\', '/')
|
||
|
lprint(f'- 7z unzip install {label} to: {exe_install_dir}', level=1)
|
||
|
lprint(f' Command: {command}', level=1)
|
||
|
subprocess.run(command)
|
||
|
|
||
|
if intermediate_zip_file_extracted:
|
||
|
lprint(f'- Detected intermediate zip file in install root, removing: {archive_path}', level=1)
|
||
|
os.remove(archive_path)
|
||
|
|
||
|
# Remove duplicate root folder if detected
|
||
|
# ----------------------------------------------------------------------
|
||
|
# If after unpacking, there's only 1 directory in the install direction, we
|
||
|
# assume that the zip contains a root folder. We will automatically merge
|
||
|
# the root folder to the parent.
|
||
|
has_files_in_install_dir = False
|
||
|
dir_count = 0
|
||
|
dupe_root_folder_name = ''
|
||
|
with os.scandir(exe_install_dir) as scan_handle:
|
||
|
for it in list(scan_handle):
|
||
|
if it.is_file():
|
||
|
has_files_in_install_dir = True
|
||
|
break
|
||
|
elif it.is_dir():
|
||
|
dupe_root_folder_name = it.name
|
||
|
dir_count += 1
|
||
|
if dir_count > 1:
|
||
|
break
|
||
|
|
||
|
if dir_count == 1 and not has_files_in_install_dir:
|
||
|
# There is only one folder after we unzipped, what happened here is
|
||
|
# that the archive we unzipped had its contents within a root
|
||
|
# folder. We will pull those files out because we already unzipped
|
||
|
# into an isolated location for the application, e.g.
|
||
|
#
|
||
|
# Our install location C:/Dev/Install/7zip/920
|
||
|
# After unzip C:/Dev/Install/7zip/920/7zip-920-x64
|
||
|
#
|
||
|
# We have an duplicate '7zip-920-x64' directory in our
|
||
|
# installation. Move all the files in the duplicate directory up to
|
||
|
# our '920' folder then remove the duplicate folder.
|
||
|
dupe_root_folder_path = pathlib.Path(exe_install_dir, dupe_root_folder_name)
|
||
|
lprint(f'- Detected duplicate root folder after unzip: {dupe_root_folder_path}', level=1)
|
||
|
lprint(f' Merging duplicate root folder to parent: {exe_install_dir}', level=1)
|
||
|
for file_name in os.listdir(dupe_root_folder_path):
|
||
|
src = pathlib.Path(dupe_root_folder_path, file_name)
|
||
|
dest = pathlib.Path(exe_install_dir, file_name)
|
||
|
shutil.move(src, dest)
|
||
|
|
||
|
os.rmdir(dupe_root_folder_path)
|
||
|
|
||
|
# Verify the installation by checking the SHA256 of the executables
|
||
|
# --------------------------------------------------------------------------
|
||
|
exes_with_bad_hashes = []
|
||
|
for exe_dict in exe_list:
|
||
|
exe_rel_path = exe_dict['path']
|
||
|
exe_path = get_exe_install_path(install_dir=install_dir,
|
||
|
label=label,
|
||
|
version_label=version_label,
|
||
|
exe_rel_path=exe_rel_path)
|
||
|
if os.path.isfile(exe_path) == False:
|
||
|
lexit(f'- Installed {label} but could not find expected file for validating install: {exe_path}', level=1)
|
||
|
|
||
|
if verify_file_sha256(file_path=exe_path, checksum=exe_dict['checksum'], label=exe_rel_path) == False:
|
||
|
exes_with_bad_hashes.append(exe_dict)
|
||
|
|
||
|
if len(exes_with_bad_hashes) > 0:
|
||
|
lprint(f'- {label} is installed but executable SHA256 does not match!', level=1)
|
||
|
lprint(f' See hashes above, executable path(s):', level=1)
|
||
|
for exe_dict in exes_with_bad_hashes:
|
||
|
lprint(f' {exe_dict["path"]}', level=1)
|
||
|
lprint(f' Something has modified the executable, this may be malicious or not!', level=1)
|
||
|
lprint(f' Manually uninstall the existing installation or amend the binary to be', level=1)
|
||
|
lprint(f' able to continue. Exiting.', level=1)
|
||
|
exit()
|
||
|
else:
|
||
|
lprint(f'- {label} installed and valid: {exe_install_dir}', level=1)
|
||
|
|
||
|
# Do the symlinks
|
||
|
# --------------------------------------------------------------------------
|
||
|
symlink_dir = get_exe_symlink_dir(install_dir)
|
||
|
paths_to_add_to_devenv_script = set()
|
||
|
for exe_dict in exe_list:
|
||
|
exe_rel_path = exe_dict['path']
|
||
|
exe_path = get_exe_install_path(install_dir=install_dir,
|
||
|
label=label,
|
||
|
version_label=version_label,
|
||
|
exe_rel_path=exe_rel_path)
|
||
|
|
||
|
for symlink_entry in exe_dict["symlink"]:
|
||
|
symlink_dest = symlink_dir / symlink_entry
|
||
|
symlink_src = exe_path
|
||
|
if os.path.exists(symlink_dest):
|
||
|
# Windows uses hardlinks because symlinks require you to enable "developer" mode
|
||
|
# Everyone else uses symlinks
|
||
|
if (IS_WINDOWS and not os.path.isfile(symlink_dest)) or (not IS_WINDOWS and not os.path.islink(symlink_dest)):
|
||
|
lprint( "- Cannot create symlink! The destionation file to create the symlink at.", level=1)
|
||
|
lprint( " already exists and is *not* a link. We cannot remove this safely as we", level=1)
|
||
|
lprint( " don't know what it is, exiting.", level=1)
|
||
|
lprint(f" Symlink Source: {symlink_src}", level=1)
|
||
|
lexit (f" Symlink Dest: {symlink_dest}", level=1)
|
||
|
|
||
|
os.unlink(symlink_dest)
|
||
|
|
||
|
if IS_WINDOWS == True:
|
||
|
os.link(src=symlink_src, dst=symlink_dest)
|
||
|
else:
|
||
|
os.symlink(src=symlink_src, dst=symlink_dest)
|
||
|
|
||
|
# Collect paths to add to the devenv script
|
||
|
# ----------------------------------------------------------------------
|
||
|
if exe_dict['add_to_devenv_path'] == True:
|
||
|
path = exe_path.parent.relative_to(install_dir)
|
||
|
paths_to_add_to_devenv_script.add(path)
|
||
|
|
||
|
global devenv_script_buffer
|
||
|
for path in paths_to_add_to_devenv_script:
|
||
|
if IS_WINDOWS:
|
||
|
devenv_script_buffer += f"set PATH=\"%~dp0{path}\";%PATH%\n"
|
||
|
else:
|
||
|
devenv_script_buffer += f"PATH=\"$( cd -- \"$( dirname -- \"${BASH_SOURCE[0]}\" )\" &> /dev/null && pwd ){path}\";%PATH%\n"
|
||
|
|
||
|
# Search the 2 dictionarries, 'first' and 'second' for the key. A matching key
|
||
|
# in 'first' taking precedence over the 'second' dictionary. If no key is
|
||
|
# found in either dictionaries then this function
|
||
|
# returns an empty string.
|
||
|
class ValidateAppListResult:
|
||
|
def __init__(self):
|
||
|
self.app_count = 0
|
||
|
|
||
|
def validate_app_list(app_list):
|
||
|
result = ValidateAppListResult()
|
||
|
manifest_rule_table = {
|
||
|
'download_checksum': 'manifest must specify the SHA256 checksum for the downloaded file',
|
||
|
'version': 'manifest must specify the app version that is to be installed',
|
||
|
'executables': 'manifest must specify an array of executable(s) for verifying installation',
|
||
|
'download_url': 'manifest must specify the URL to download the app from',
|
||
|
'add_to_devenv_script': 'manifest must specify an array of strings to inject into the portable development environment setup script',
|
||
|
}
|
||
|
|
||
|
executable_rule_table = {
|
||
|
'path': 'executables must specify a path to a file from the installation to verify its checksum',
|
||
|
'symlink': 'executables must specify an array of symlink names that will target the path',
|
||
|
'add_to_devenv_path': 'executables must specify an boolean to indicate if the executable path should be added to the environment path',
|
||
|
'checksum': 'executables must specify a string with the checksum of the executable',
|
||
|
}
|
||
|
|
||
|
for app in app_list:
|
||
|
manifest_list = app['manifests']
|
||
|
result.app_count += len(manifest_list)
|
||
|
|
||
|
# Verify the label
|
||
|
# ----------------------------------------------------------------------
|
||
|
label = app.get('label', '')
|
||
|
if 'label' not in app:
|
||
|
exit('Label missing from application list, app must have a label specified, e.g. { "label": "App Name", "manifests": [] }')
|
||
|
|
||
|
# Verify that the mandatory keys are in the manifest
|
||
|
# ----------------------------------------------------------------------
|
||
|
for manifest in manifest_list:
|
||
|
for key in manifest_rule_table:
|
||
|
value = manifest.get(key, "")
|
||
|
|
||
|
if key.startswith("add_to_devenv"):
|
||
|
if not isinstance(value, list):
|
||
|
exit(f'{label} error: {key} in manifest must be an array to proceed\n{pprint.pformat(app)}')
|
||
|
|
||
|
elif key == "executables":
|
||
|
for executable in value:
|
||
|
for executable_key in executable_rule_table:
|
||
|
executable_value = executable.get(executable_key, "")
|
||
|
|
||
|
if executable_key == "path":
|
||
|
if not isinstance(executable_value, str) or len(executable_value) == 0:
|
||
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
||
|
|
||
|
elif executable_key == "symlink":
|
||
|
if not isinstance(executable_value, list):
|
||
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
||
|
|
||
|
elif executable_key == "add_to_devenv_path":
|
||
|
if not isinstance(executable_value, bool):
|
||
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
||
|
|
||
|
elif executable_key == "checksum":
|
||
|
if not isinstance(executable_value, str):
|
||
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
||
|
|
||
|
elif len(value) == 0:
|
||
|
exit(f'{label} error: required key "{key}" is missing/empty, {manifest_rule_table[key]}\n{pprint.pformat(app)}')
|
||
|
|
||
|
return result
|
||
|
|
||
|
devenv_script_buffer = """@echo off
|
||
|
|
||
|
"""
|
||
|
def install_app_list(app_list, download_dir, install_dir):
|
||
|
title = "Internal Apps" if app_list is internal_app_list else "User Apps"
|
||
|
print_header(title)
|
||
|
result = {}
|
||
|
|
||
|
validate_app_list_result = validate_app_list(app_list)
|
||
|
app_index = 0
|
||
|
|
||
|
for app in app_list:
|
||
|
manifest_list = app['manifests']
|
||
|
for manifest in manifest_list:
|
||
|
app_index += 1
|
||
|
|
||
|
# Extract variables from manifest
|
||
|
# ------------------------------------------------------------------
|
||
|
label = app["label"]
|
||
|
download_checksum = manifest['download_checksum']
|
||
|
version = manifest["version"]
|
||
|
download_url = manifest["download_url"]
|
||
|
exe_list = manifest['executables']
|
||
|
unzip_method = UnzipMethod.DEFAULT
|
||
|
|
||
|
# Bootstrapping code, when installing the internal app list, we will
|
||
|
# assign the variables to point to our unarchiving tools.
|
||
|
# ------------------------------------------------------------------
|
||
|
if app_list is internal_app_list:
|
||
|
global zip7_exe
|
||
|
global zip7_bootstrap_exe
|
||
|
global zstd_exe
|
||
|
exe_path = get_exe_install_path(install_dir, label, version, manifest['executables'][0]['path'])
|
||
|
if label == '7zip':
|
||
|
if version == '920':
|
||
|
unzip_method = UnzipMethod.SHUTILS
|
||
|
zip7_bootstrap_exe = exe_path
|
||
|
else:
|
||
|
unzip_method = UnzipMethod.ZIP7_BOOTSTRAP
|
||
|
zip7_exe = exe_path
|
||
|
|
||
|
if label == 'zstd':
|
||
|
zstd_exe = exe_path
|
||
|
|
||
|
|
||
|
# Download and install
|
||
|
# ------------------------------------------------------------------
|
||
|
lprint(f'[{app_index:03}/{validate_app_list_result.app_count:03}] Setup {label} v{version}', level=0)
|
||
|
download_and_install_archive(download_url=download_url,
|
||
|
download_checksum=download_checksum,
|
||
|
exe_list=exe_list,
|
||
|
version_label=version,
|
||
|
label=label,
|
||
|
unzip_method=unzip_method,
|
||
|
download_dir=download_dir,
|
||
|
install_dir=install_dir)
|
||
|
|
||
|
# Post-installation
|
||
|
# ------------------------------------------------------------------
|
||
|
|
||
|
# Collate results into results
|
||
|
if label not in result:
|
||
|
result.update({label: []})
|
||
|
|
||
|
for item in exe_list:
|
||
|
app_install_dir = get_exe_install_dir(install_dir, label, version)
|
||
|
app_exe_path = get_exe_install_path(install_dir, label, version, item['path'])
|
||
|
app_exe_dir = pathlib.Path(app_exe_path).parent
|
||
|
|
||
|
# Add executable into the result list
|
||
|
result[label].append({
|
||
|
'version': version,
|
||
|
'install_dir': app_install_dir,
|
||
|
'exe_path': app_exe_path,
|
||
|
})
|
||
|
|
||
|
# Add the snippets verbatim specified in the manifest
|
||
|
global devenv_script_buffer
|
||
|
for line in manifest['add_to_devenv_script']:
|
||
|
devenv_script_buffer += (line + '\n')
|
||
|
|
||
|
if app_list is internal_app_list:
|
||
|
if len(str(zip7_exe)) == 0 or len(str(zip7_bootstrap_exe)) == 0 or len(str(zstd_exe)) == 0:
|
||
|
exit("Internal app list did not install 7zip bootstrap, 7zip or zstd, we are unable to install archives\n"
|
||
|
f" - zip7_bootstrap_exe: {zip7_bootstrap_exe}\n"
|
||
|
f" - zip7_exe: {zip7_exe}\n"
|
||
|
f" - zstd_exe: {zstd_exe}\n")
|
||
|
|
||
|
return result
|
||
|
|
||
|
def run(user_app_list,
|
||
|
download_dir=base_downloads_dir,
|
||
|
install_dir=base_install_dir):
|
||
|
""" Download and install the given user app list at the specified
|
||
|
directories. The apps given must be archives that can be unpacked for
|
||
|
installation (e.g. portable distributions).
|
||
|
|
||
|
Parameters:
|
||
|
user_app_list (list): A list of dictionaries that contain app and
|
||
|
manifest information dictating what is to be installed via this
|
||
|
function.
|
||
|
download_dir (string): The path that intermediate downloaded files will
|
||
|
be kept at.
|
||
|
install_dir (string): The path that installed applications will be
|
||
|
unpacked to
|
||
|
|
||
|
Returns:
|
||
|
result (list): A list of dictionaries containing the install locations
|
||
|
of each app, e.g.
|
||
|
"""
|
||
|
# Run
|
||
|
# --------------------------------------------------------------------------
|
||
|
# Create the starting directories and install the internal app list (e.g.
|
||
|
# 7zip) which will be used to unzip-install the rest of the apps in the user
|
||
|
# app list.
|
||
|
#
|
||
|
# To do this without dependencies, we first download an old version of 7zip,
|
||
|
# version 9.20 which is distributed as a .zip file which Python can natively
|
||
|
# unzip.
|
||
|
#
|
||
|
# We then use the old version of 7zip and download a newer version of 7zip
|
||
|
# and extract it using the bootstrap-ed version. As of writing this, 7zip
|
||
|
# does not release a portable distribution of itself yet, instead what we do
|
||
|
# is download the installer and extract it ourselves using the bootstrap
|
||
|
# 7zip.
|
||
|
|
||
|
# Create the paths requested by the user
|
||
|
os.makedirs(download_dir, exist_ok=True)
|
||
|
os.makedirs(install_dir, exist_ok=True)
|
||
|
os.makedirs(pathlib.Path(install_dir, "Symlinks"), exist_ok=True)
|
||
|
|
||
|
for path in [download_dir, install_dir]:
|
||
|
if not os.path.isdir(path):
|
||
|
exit(f'Path "{path}" is not a directory, script can not proceed. Exiting.')
|
||
|
|
||
|
# Validate all the manifests before starting
|
||
|
internal_app_validate_result = validate_app_list(internal_app_list)
|
||
|
user_app_validate_result = validate_app_list(user_app_list)
|
||
|
|
||
|
# Install apps
|
||
|
internal_apps = install_app_list(app_list=internal_app_list,
|
||
|
download_dir=download_dir,
|
||
|
install_dir=install_dir)
|
||
|
|
||
|
user_apps = install_app_list(app_list=user_app_list,
|
||
|
download_dir=download_dir,
|
||
|
install_dir=install_dir)
|
||
|
|
||
|
# Write the devenv script with environment variables
|
||
|
global devenv_script_buffer
|
||
|
devenv_script_buffer += "set PATH=\"%~dp0Symlinks\";%PATH%\n"
|
||
|
|
||
|
devenv_script_name = "devenv.bat" if IS_WINDOWS else "devenv.sh"
|
||
|
devenv_script_path = pathlib.Path(install_dir, devenv_script_name)
|
||
|
lprint(f"Writing script to augment the environment with installed applications: {devenv_script_path}")
|
||
|
with open(devenv_script_path, 'w') as file:
|
||
|
file.write(devenv_script_buffer)
|
||
|
|
||
|
# Merge the install dictionaries, this dictionary contains
|
||
|
# (app label) -> [array of installed versions]
|
||
|
result = internal_apps
|
||
|
for key, value in user_apps.items():
|
||
|
if key not in result:
|
||
|
result.update({key: value})
|
||
|
else:
|
||
|
result[key] += value
|
||
|
|
||
|
return result
|
||
|
|
||
|
if __name__ == '__main__':
|
||
|
run()
|