840 lines
38 KiB
Python
840 lines
38 KiB
Python
#!/usr/bin/env python3
|
|
|
|
# DEVenver
|
|
# ------------------------------------------------------------------------------
|
|
# A simple python script to download portable applications and install them by
|
|
# unzipping them to a structured directory tree.
|
|
|
|
import urllib.request
|
|
import urllib.parse
|
|
import pathlib
|
|
import os
|
|
import tempfile
|
|
import hashlib
|
|
import shutil
|
|
import subprocess
|
|
import pprint
|
|
import argparse
|
|
import json
|
|
import importlib
|
|
|
|
from string import Template
|
|
from enum import Enum
|
|
|
|
# Internal
|
|
# ------------------------------------------------------------------------------
|
|
DOWNLOAD_CHUNK_SIZE = 1 * 1024 * 1024 # 1 megabyte
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
# These variables are set once they are downloaded dynamically and installed
|
|
# from the internal app listing!
|
|
zstd_exe = ""
|
|
zip7_exe = ""
|
|
zip7_bootstrap_exe = ""
|
|
|
|
# Functions
|
|
# ------------------------------------------------------------------------------
|
|
def print_header(title):
|
|
line = f'> ' + title + ' ';
|
|
print(line.ljust(100, '-'))
|
|
|
|
def lprint(*args, level=0, **kwargs):
|
|
print(' ' + (' ' * 2 * level), *args, **kwargs)
|
|
|
|
def lexit(*args, level=0, **kwargs):
|
|
print(' ' + (' ' * 2 * level), *args, **kwargs)
|
|
exit()
|
|
|
|
def verify_file_sha256(file_path, checksum, label):
|
|
if os.path.isfile(file_path) == False:
|
|
exit(f'Cannot verify SHA256, path is not a file [path={file_path}]')
|
|
|
|
result = False
|
|
try:
|
|
file = open(file_path, 'rb')
|
|
hasher = hashlib.sha256()
|
|
hasher.update(file.read())
|
|
derived_checksum = hasher.hexdigest()
|
|
result = derived_checksum == checksum
|
|
if result:
|
|
lprint(f'- {label} SHA256 is good: {checksum}', level=1)
|
|
else:
|
|
lprint(f'- {label} SHA256 mismatch', level=1)
|
|
lprint(f' Expect: {checksum}', level=1)
|
|
lprint(f' Actual: {derived_checksum}', level=1)
|
|
except PermissionError as exception:
|
|
lprint(f"- {label} cannot verify SHA256 due to permission error, skipping", level=1)
|
|
result = True
|
|
else:
|
|
file.close()
|
|
|
|
return result
|
|
|
|
def download_file_at_url(url, download_path, download_checksum, label):
|
|
# Check if file already downloaded and hashes match
|
|
# --------------------------------------------------------------------------
|
|
file_already_downloaded = False
|
|
if os.path.isfile(download_path):
|
|
lprint(f'- Cached archive found: {download_path}', level=1)
|
|
file_already_downloaded = verify_file_sha256(download_path, download_checksum, 'Cached archive')
|
|
else:
|
|
lprint(f'- Download to disk: {download_path}', level=1)
|
|
lprint(f' URL: {url}', level=1)
|
|
|
|
# Download the file from URL
|
|
# --------------------------------------------------------------------------
|
|
if file_already_downloaded == False:
|
|
lprint('Initiating download request ...', level=2)
|
|
with urllib.request.urlopen(url) as response:
|
|
temp_file = tempfile.mkstemp(text=False)
|
|
temp_file_handle = temp_file[0]
|
|
temp_file_path = temp_file[1]
|
|
temp_file_io = os.fdopen(temp_file_handle, mode='w+b')
|
|
|
|
download_failed = False
|
|
try:
|
|
line = ''
|
|
total_download_size = int(response.getheader('Content-Length'))
|
|
bytes_downloaded = 0
|
|
while chunk := response.read(DOWNLOAD_CHUNK_SIZE):
|
|
bytes_written = temp_file_io.write(chunk)
|
|
bytes_downloaded += bytes_written
|
|
percent_downloaded = int(bytes_downloaded / total_download_size * 100)
|
|
|
|
lprint(' ' * len(line), end='\r', level=2)
|
|
line = f'Downloading {percent_downloaded:.2f}% ({bytes_downloaded}/{total_download_size})'
|
|
lprint(line, end='\r', level=2)
|
|
except Exception as exception:
|
|
download_failed = True
|
|
lprint(f'Download {label} from {url} failed, {exception}', level=2)
|
|
finally:
|
|
temp_file_io.close()
|
|
print()
|
|
|
|
if download_failed == True:
|
|
os.remove(temp_file_path)
|
|
exit()
|
|
|
|
# If file already exists then we have a hash mismatch, delete it
|
|
# then rename the new item ontop of it
|
|
if os.path.isfile(download_path):
|
|
os.unlink(download_path)
|
|
|
|
shutil.move(temp_file_path, download_path)
|
|
|
|
if file_already_downloaded == False:
|
|
if verify_file_sha256(download_path, download_checksum, 'Downloaded archive') == False:
|
|
exit()
|
|
|
|
class UnzipMethod(Enum):
|
|
SHUTILS = 0
|
|
ZIP7_BOOTSTRAP = 1
|
|
DEFAULT = 2
|
|
ZIP7 = 3
|
|
NO_UNZIP = 4
|
|
|
|
def get_exe_install_dir(install_dir, label, version_label):
|
|
result = pathlib.Path(install_dir, label.replace(' ', '_'), version_label)
|
|
return result
|
|
|
|
def get_exe_install_path(install_dir, label, version_label, exe_rel_path):
|
|
install_dir = get_exe_install_dir(install_dir, label, version_label)
|
|
result = pathlib.Path(install_dir, exe_rel_path)
|
|
return result
|
|
|
|
def get_exe_symlink_dir(install_dir):
|
|
result = pathlib.Path(install_dir, "Symlinks")
|
|
return result
|
|
|
|
def download_and_install_archive(download_url,
|
|
download_checksum,
|
|
exe_list,
|
|
version_label,
|
|
label,
|
|
unzip_method,
|
|
download_dir,
|
|
install_dir,
|
|
is_windows):
|
|
|
|
exe_install_dir = get_exe_install_dir(install_dir=install_dir,
|
|
label=label,
|
|
version_label=version_label)
|
|
|
|
# Evaluate if we have already installed the requested archive
|
|
# --------------------------------------------------------------------------
|
|
exes_are_not_a_file = []
|
|
exes_missing = []
|
|
exes_present = []
|
|
for exe_dict in exe_list:
|
|
exe_path = get_exe_install_path(install_dir=install_dir,
|
|
label=label,
|
|
version_label=version_label,
|
|
exe_rel_path=exe_dict['path'])
|
|
|
|
if os.path.exists(exe_path) == True:
|
|
if os.path.isfile(exe_path) == True:
|
|
exes_present.append(exe_dict)
|
|
else:
|
|
exes_are_not_a_file.append(exe_dict)
|
|
else:
|
|
exes_missing.append(exe_dict)
|
|
|
|
# Executables not install yet, verify, download and install if possible
|
|
# --------------------------------------------------------------------------
|
|
if len(exes_present) != len(exe_list):
|
|
|
|
# Check if any of the manifest files are not files
|
|
# ----------------------------------------------------------------------
|
|
if len(exes_are_not_a_file) > 0: # Some item exists at the path but they are not files
|
|
lprint(f'- {label} is installed but some of the expected executables are not a file!', level=1)
|
|
for exe_dict in exes_are_not_a_file:
|
|
lprint(f' {exe_dict["path"]}', level=1)
|
|
lprint(f' Installation cannot proceed as unpacking would overwrite these paths', level=1)
|
|
exit()
|
|
|
|
# Check if any files are missing
|
|
# ----------------------------------------------------------------------
|
|
# Some executables are missing, some are available, installation will
|
|
# trample over existing files, its not safe to unzip the archive as we may
|
|
# overwrite config files or some other files that have been modified by the
|
|
# program.
|
|
#
|
|
# Note that all files missing means we can assume that we haven't installed
|
|
# yet ..
|
|
if len(exes_missing) > 0 and len(exes_missing) != len(exe_list):
|
|
lprint(f'- {label} is installed but some of the expected executables are missing from the installation!', level=1)
|
|
for exe_dict in exes_are_not_a_file:
|
|
lprint(f' {exe_dict["path"]}', level=1)
|
|
exit()
|
|
|
|
assert(len(exes_missing) == len(exe_list))
|
|
assert(len(exes_present) == 0)
|
|
|
|
# Not installed yet, download and install
|
|
# ----------------------------------------------------------------------
|
|
# Determine the file name we are downloading from the URL
|
|
download_url_parts = urllib.parse.urlparse(download_url)
|
|
download_name = pathlib.Path(urllib.parse.unquote(download_url_parts.path))
|
|
|
|
# The path to move the temp file to after successful download, e.g.
|
|
# download_dir = C:/Dev/Downloads/Wezterm-windows-{version}.zip
|
|
# download_name = Wezterm-windows-{version}.zip
|
|
download_path = pathlib.Path(download_dir, download_name.name)
|
|
|
|
# Download the archive at the URL
|
|
download_file_at_url(download_url, download_path, download_checksum, label)
|
|
|
|
# Install the archive by unpacking it
|
|
# ----------------------------------------------------------------------
|
|
archive_path = download_path
|
|
os.makedirs(exe_install_dir, exist_ok=True)
|
|
|
|
if unzip_method == UnzipMethod.DEFAULT:
|
|
if archive_path.suffix == '.exe' or archive_path.suffix.lower() == '.appimage' or len(archive_path.suffix) == 0:
|
|
unzip_method = UnzipMethod.NO_UNZIP
|
|
|
|
if unzip_method == UnzipMethod.NO_UNZIP:
|
|
shutil.copy(archive_path, exe_install_dir)
|
|
elif unzip_method == UnzipMethod.SHUTILS:
|
|
lprint(f'- SHUtils unzip install {label} to: {exe_install_dir}', level=1)
|
|
shutil.unpack_archive(download_path, exe_install_dir)
|
|
elif unzip_method == UnzipMethod.ZIP7_BOOTSTRAP:
|
|
command = [str(zip7_bootstrap_exe), "x", "-bd", str(download_path), f"-o{exe_install_dir}"]
|
|
lprint(f'- 7z (bootstrap) unzip {label} to: {exe_install_dir}', level=1)
|
|
lprint(f' Command: {command}', level=1)
|
|
subprocess.run(command)
|
|
else: # Default or 7ZIP
|
|
intermediate_zip_file_extracted = False
|
|
|
|
# We could have a "app.zst" situation or an "app.tar.zst" situation
|
|
#
|
|
# "app.zst" only needs 1 extraction from the zstd tool
|
|
# "app.tar.zst" needs 1 zstd extract and then 1 7zip extract
|
|
#
|
|
# When we have "app.tar.zst" we extract to the install folder, e.g.
|
|
#
|
|
# "app/1.0/app.tar"
|
|
#
|
|
# We call this an intermediate zip file, we will extract that file
|
|
# with 7zip. After we're done, we will delete that _intermediate_
|
|
# file to cleanup our install directory.
|
|
unzip_command = [str(zip7_exe), "x", "-aoa", "-spe", "-bso0", str(archive_path), f"-o{exe_install_dir}"]
|
|
|
|
linux_used_tar = False
|
|
if archive_path.suffix == '.zst' or archive_path.suffix == '.xz' or archive_path.suffix == '.gz' or archive_path.suffix == '.bz2':
|
|
archive_without_suffix = pathlib.Path(str(archive_path)[:-len(archive_path.suffix)]).name
|
|
next_archive_path = pathlib.Path(exe_install_dir, archive_without_suffix)
|
|
|
|
if os.path.exists(next_archive_path) == False:
|
|
command = ""
|
|
if archive_path.suffix == '.zst':
|
|
command = [str(zstd_exe), "-d", "-f", str(archive_path)]
|
|
lprint(f'- zstd unzip {label}', level=1)
|
|
|
|
# zstd extracts to the same directory as the archive
|
|
next_archive_path = pathlib.Path(download_dir, archive_without_suffix)
|
|
else:
|
|
if not is_windows:
|
|
linux_used_tar = True
|
|
unzip_command = ["tar", "xf", str(archive_path), "--directory", str(exe_install_dir)]
|
|
|
|
command = unzip_command
|
|
for item in command:
|
|
item = item.replace('\\', '/')
|
|
lprint(f'- Unzip nested install {label} to: {exe_install_dir}', level=1)
|
|
|
|
lprint(f' Command: {command}', level=1)
|
|
subprocess.run(args=command)
|
|
|
|
# Remove the extension from the file, we just extracted it
|
|
archive_path = next_archive_path
|
|
|
|
# If there's still a suffix after we removed the ".zst" we got
|
|
# an additional archive to unzip, e.g. "app.tar" remaining.
|
|
if not linux_used_tar:
|
|
intermediate_zip_file_extracted = len(archive_path.suffix) > 0
|
|
|
|
# \note On Linux using tar xf will unpack the archive entirely, e.g.
|
|
# unpack tar.gz both in one shot. So we do not need a further
|
|
# extract step.
|
|
if not linux_used_tar and len(archive_path.suffix) > 0:
|
|
unzip_command = [str(zip7_exe), "x", "-aoa", "-spe", "-bso0", str(archive_path), f"-o{exe_install_dir}"]
|
|
command = unzip_command
|
|
for item in command:
|
|
item = item.replace('\\', '/')
|
|
|
|
lprint(f'- Unzip install {label} to: {exe_install_dir}', level=1)
|
|
lprint(f' Command: {command}', level=1)
|
|
subprocess.run(command)
|
|
|
|
if intermediate_zip_file_extracted:
|
|
lprint(f'- Detected intermediate zip file in install root, removing: {archive_path}', level=1)
|
|
os.remove(archive_path)
|
|
|
|
# Remove duplicate root folder if detected
|
|
# ----------------------------------------------------------------------
|
|
# If after unpacking, there's only 1 directory in the install direction, we
|
|
# assume that the zip contains a root folder. We will automatically merge
|
|
# the root folder to the parent.
|
|
has_files_in_install_dir = False
|
|
dir_count = 0
|
|
dupe_root_folder_name = ''
|
|
with os.scandir(exe_install_dir) as scan_handle:
|
|
for it in list(scan_handle):
|
|
if it.is_file():
|
|
has_files_in_install_dir = True
|
|
break
|
|
elif it.is_dir():
|
|
dupe_root_folder_name = it.name
|
|
dir_count += 1
|
|
if dir_count > 1:
|
|
break
|
|
|
|
if dir_count == 1 and not has_files_in_install_dir:
|
|
# There is only one folder after we unzipped, what happened here is
|
|
# that the archive we unzipped had its contents within a root
|
|
# folder. We will pull those files out because we already unzipped
|
|
# into an isolated location for the application, e.g.
|
|
#
|
|
# Our install location C:/Dev/Install/7zip/920
|
|
# After unzip C:/Dev/Install/7zip/920/7zip-920-x64
|
|
#
|
|
# We have an duplicate '7zip-920-x64' directory in our
|
|
# installation. Move all the files in the duplicate directory up to
|
|
# our '920' folder then remove the duplicate folder.
|
|
dupe_root_folder_path = pathlib.Path(exe_install_dir, dupe_root_folder_name)
|
|
lprint(f'- Detected duplicate root folder after unzip: {dupe_root_folder_path}', level=1)
|
|
lprint(f' Merging duplicate root folder to parent: {exe_install_dir}', level=1)
|
|
for file_name in os.listdir(dupe_root_folder_path):
|
|
src = pathlib.Path(dupe_root_folder_path, file_name)
|
|
dest = pathlib.Path(exe_install_dir, file_name)
|
|
shutil.move(src, dest)
|
|
|
|
os.rmdir(dupe_root_folder_path)
|
|
|
|
# Verify the installation by checking the SHA256 of the executables
|
|
# --------------------------------------------------------------------------
|
|
exes_with_bad_hashes = []
|
|
for exe_dict in exe_list:
|
|
exe_rel_path = exe_dict['path']
|
|
exe_path = get_exe_install_path(install_dir=install_dir,
|
|
label=label,
|
|
version_label=version_label,
|
|
exe_rel_path=exe_rel_path)
|
|
if os.path.isfile(exe_path) == False:
|
|
lexit(f'- Installed {label} but could not find expected file for validating install: {exe_path}', level=1)
|
|
|
|
if verify_file_sha256(file_path=exe_path, checksum=exe_dict['checksum'], label=exe_rel_path) == False:
|
|
exes_with_bad_hashes.append(exe_dict)
|
|
|
|
if len(exes_with_bad_hashes) > 0:
|
|
lprint(f'- {label} is installed but executable SHA256 does not match!', level=1)
|
|
lprint(f' See hashes above, executable path(s):', level=1)
|
|
for exe_dict in exes_with_bad_hashes:
|
|
lprint(f' {exe_dict["path"]}', level=1)
|
|
lprint(f' Something has modified the executable, this may be malicious or not!', level=1)
|
|
lprint(f' Manually uninstall the existing installation or amend the binary to be', level=1)
|
|
lprint(f' able to continue. Exiting.', level=1)
|
|
exit()
|
|
else:
|
|
lprint(f'- {label} installed and valid: {exe_install_dir}', level=1)
|
|
|
|
# Do the symlinks
|
|
# --------------------------------------------------------------------------
|
|
symlink_dir = get_exe_symlink_dir(install_dir)
|
|
paths_to_add_to_devenv_script = set()
|
|
for exe_dict in exe_list:
|
|
exe_rel_path = exe_dict['path']
|
|
exe_path = get_exe_install_path(install_dir=install_dir,
|
|
label=label,
|
|
version_label=version_label,
|
|
exe_rel_path=exe_rel_path)
|
|
# Make them executable
|
|
# ----------------------------------------------------------------------
|
|
if not is_windows:
|
|
subprocess.run(args=["chmod", "+x", exe_path])
|
|
|
|
# If you install the Linux manifest on Windows then we ensure we
|
|
# still call link (e.g. hardlink) because symlinks need special
|
|
# Windows 10 permissions to be set...
|
|
#
|
|
# It's weird that we install Linux manifest on Windows, yes, but
|
|
# I use it for testing without having to boot up a whole nother
|
|
# OS.
|
|
use_hardlink = is_windows or os.name == 'nt'
|
|
for symlink_entry in exe_dict["symlink"]:
|
|
symlink_dest = symlink_dir / symlink_entry
|
|
symlink_src = exe_path
|
|
skip_link = False;
|
|
if os.path.exists(symlink_dest):
|
|
# Windows uses hardlinks because symlinks require you to enable "developer" mode
|
|
# Everyone else uses symlinks
|
|
if (use_hardlink and not os.path.isfile(symlink_dest)) or (not use_hardlink and not os.path.islink(symlink_dest)):
|
|
lprint( "- Cannot create symlink! The destination file to create the symlink at.", level=1)
|
|
lprint( " already exists and is *not* a link. We cannot remove this safely as we", level=1)
|
|
lprint( " don't know what it is, exiting.", level=1)
|
|
lprint(f" Symlink Source: {symlink_src}", level=1)
|
|
lexit (f" Symlink Dest: {symlink_dest}", level=1)
|
|
|
|
if os.path.samefile(symlink_dest, symlink_src):
|
|
skip_link = True
|
|
else:
|
|
os.unlink(symlink_dest)
|
|
|
|
if not skip_link:
|
|
if use_hardlink:
|
|
os.link(src=symlink_src, dst=symlink_dest)
|
|
else:
|
|
os.symlink(src=symlink_src, dst=symlink_dest)
|
|
|
|
# Collect paths to add to the devenv script
|
|
# ----------------------------------------------------------------------
|
|
if exe_dict['add_to_devenv_path'] == True:
|
|
path = exe_path.parent.relative_to(install_dir)
|
|
paths_to_add_to_devenv_script.add(path)
|
|
|
|
global devenv_script_buffer
|
|
for path in paths_to_add_to_devenv_script:
|
|
if is_windows:
|
|
devenv_script_buffer += f"set PATH=%~dp0{path};%PATH%\n"
|
|
else:
|
|
devenv_script_buffer += f"PATH=\"$( cd -- $( dirname -- \"${{BASH_SOURCE[0]}}\" ) &> /dev/null && pwd )/{path}\":$PATH\n"
|
|
|
|
# Search the 2 dictionarries, 'first' and 'second' for the key. A matching key
|
|
# in 'first' taking precedence over the 'second' dictionary. If no key is
|
|
# found in either dictionaries then this function
|
|
# returns an empty string.
|
|
class ValidateAppListResult:
|
|
def __init__(self):
|
|
self.app_count = 0
|
|
|
|
def validate_app_list(app_list):
|
|
result = ValidateAppListResult()
|
|
manifest_rule_table = {
|
|
'download_checksum': 'manifest must specify the SHA256 checksum for the downloaded file',
|
|
'version': 'manifest must specify the app version that is to be installed',
|
|
'executables': 'manifest must specify an array of executable(s) for verifying installation',
|
|
'download_url': 'manifest must specify the URL to download the app from',
|
|
'add_to_devenv_script': 'manifest must specify an array of strings to inject into the portable development environment setup script',
|
|
}
|
|
|
|
executable_rule_table = {
|
|
'path': 'executables must specify a path to a file from the installation to verify its checksum',
|
|
'symlink': 'executables must specify an array of symlink names that will target the path',
|
|
'add_to_devenv_path': 'executables must specify an boolean to indicate if the executable path should be added to the environment path',
|
|
'checksum': 'executables must specify a string with the checksum of the executable',
|
|
}
|
|
|
|
for app in app_list:
|
|
manifest_list = app['manifests']
|
|
result.app_count += len(manifest_list)
|
|
|
|
# Verify the label
|
|
# ----------------------------------------------------------------------
|
|
label = app.get('label', '')
|
|
if 'label' not in app:
|
|
exit('Label missing from application list, app must have a label specified, e.g. { "label": "App Name", "manifests": [] }')
|
|
|
|
# Verify that the mandatory keys are in the manifest
|
|
# ----------------------------------------------------------------------
|
|
for manifest in manifest_list:
|
|
for key in manifest_rule_table:
|
|
value = manifest.get(key, "")
|
|
|
|
if key.startswith("add_to_devenv"):
|
|
if not isinstance(value, list):
|
|
exit(f'{label} error: {key} in manifest must be an array to proceed\n{pprint.pformat(app)}')
|
|
|
|
elif key == "executables":
|
|
for executable in value:
|
|
for executable_key in executable_rule_table:
|
|
executable_value = executable.get(executable_key, "")
|
|
|
|
if executable_key == "path":
|
|
if not isinstance(executable_value, str) or len(executable_value) == 0:
|
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
|
|
|
elif executable_key == "symlink":
|
|
if not isinstance(executable_value, list):
|
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
|
|
|
elif executable_key == "add_to_devenv_path":
|
|
if not isinstance(executable_value, bool):
|
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
|
|
|
elif executable_key == "checksum":
|
|
if not isinstance(executable_value, str):
|
|
exit(f'{label} error: required key "{executable_key}" is invalid, {executable_rule_table[executable_key]}\n{pprint.pformat(app)}')
|
|
|
|
elif len(value) == 0:
|
|
exit(f'{label} error: required key "{key}" is missing/empty, {manifest_rule_table[key]}\n{pprint.pformat(app)}')
|
|
|
|
return result
|
|
|
|
internal_app_list = []
|
|
devenv_script_buffer = ""
|
|
|
|
def install_app_list(app_list, download_dir, install_dir, is_windows):
|
|
title = "Internal Apps" if app_list is internal_app_list else "User Apps"
|
|
print_header(title)
|
|
result = {}
|
|
|
|
validate_app_list_result = validate_app_list(app_list)
|
|
app_index = 0
|
|
|
|
global zstd_exe
|
|
if not is_windows:
|
|
zstd_exe = "/usr/bin/zstd"
|
|
|
|
for app in app_list:
|
|
manifest_list = app['manifests']
|
|
for manifest in manifest_list:
|
|
app_index += 1
|
|
|
|
# Extract variables from manifest
|
|
# ------------------------------------------------------------------
|
|
label = app["label"]
|
|
download_checksum = manifest['download_checksum']
|
|
version = manifest["version"]
|
|
download_url = manifest["download_url"]
|
|
exe_list = manifest['executables']
|
|
|
|
unzip_method = UnzipMethod.DEFAULT
|
|
unzip_method_string = manifest.get("unzip_method", "").lower()
|
|
if unzip_method_string == 'shutils':
|
|
unzip_method = UnzipMethod.SHUTILS
|
|
elif unzip_method_string == '7zip bootstrap':
|
|
unzip_method = UnzipMethod.ZIP7_BOOTSTRAP
|
|
elif unzip_method_string == 'default':
|
|
unzip_method = UnzipMethod.DEFAULT
|
|
elif unzip_method_string == '7zip':
|
|
unzip_method = UnzipMethod.ZIP7
|
|
elif unzip_method_string == 'no unzip':
|
|
unzip_method = UnzipMethod.NO_UNZIP
|
|
|
|
# Bootstrapping code, when installing the internal app list, we will
|
|
# assign the variables to point to our unarchiving tools.
|
|
# ------------------------------------------------------------------
|
|
if app_list is internal_app_list:
|
|
global zip7_exe
|
|
global zip7_bootstrap_exe
|
|
exe_path = get_exe_install_path(install_dir, label, version, manifest['executables'][0]['path'])
|
|
if label == '7zip':
|
|
if is_windows or os.name == 'nt':
|
|
if version == '920':
|
|
unzip_method = UnzipMethod.SHUTILS
|
|
zip7_bootstrap_exe = exe_path
|
|
else:
|
|
unzip_method = UnzipMethod.ZIP7_BOOTSTRAP
|
|
zip7_exe = exe_path
|
|
else:
|
|
unzip_method = UnzipMethod.SHUTILS
|
|
zip7_exe = exe_path
|
|
zip7_bootstrap_exe = exe_path
|
|
elif label == 'zstd':
|
|
zstd_exe = exe_path
|
|
|
|
# Download and install
|
|
# ------------------------------------------------------------------
|
|
lprint(f'[{app_index:03}/{validate_app_list_result.app_count:03}] Setup {label} v{version}', level=0)
|
|
download_and_install_archive(download_url=download_url,
|
|
download_checksum=download_checksum,
|
|
exe_list=exe_list,
|
|
version_label=version,
|
|
label=label,
|
|
unzip_method=unzip_method,
|
|
download_dir=download_dir,
|
|
install_dir=install_dir,
|
|
is_windows=is_windows)
|
|
|
|
# Post-installation
|
|
# ------------------------------------------------------------------
|
|
|
|
# Collate results into results
|
|
if label not in result:
|
|
result.update({label: []})
|
|
|
|
for item in exe_list:
|
|
app_install_dir = get_exe_install_dir(install_dir, label, version)
|
|
app_exe_path = get_exe_install_path(install_dir, label, version, item['path'])
|
|
app_exe_dir = pathlib.Path(app_exe_path).parent
|
|
|
|
# Add executable into the result list
|
|
result[label].append({
|
|
'version': version,
|
|
'install_dir': app_install_dir,
|
|
'exe_path': app_exe_path,
|
|
})
|
|
|
|
# Add the snippets verbatim specified in the manifest
|
|
global devenv_script_buffer
|
|
for line in manifest['add_to_devenv_script']:
|
|
devenv_script_buffer += (line + '\n')
|
|
|
|
if (is_windows or os.name == 'nt') and app_list is internal_app_list:
|
|
if len(str(zip7_exe)) == 0 or len(str(zip7_bootstrap_exe)) == 0 or len(str(zstd_exe)) == 0:
|
|
exit("Internal app list did not install 7zip bootstrap, 7zip or zstd, we are unable to install archives\n"
|
|
f" - zip7_bootstrap_exe: {zip7_bootstrap_exe}\n"
|
|
f" - zip7_exe: {zip7_exe}\n"
|
|
f" - zstd_exe: {zstd_exe}\n")
|
|
|
|
return result
|
|
|
|
script_dir = pathlib.Path(os.path.dirname(os.path.abspath(__file__)))
|
|
default_base_dir = script_dir
|
|
default_base_downloads_dir = default_base_dir / 'Downloads'
|
|
default_base_install_dir = default_base_dir / 'Install'
|
|
|
|
base_downloads_dir = default_base_downloads_dir
|
|
base_install_dir = default_base_install_dir
|
|
|
|
def run(user_app_list,
|
|
devenv_script_name,
|
|
is_windows,
|
|
download_dir=base_downloads_dir,
|
|
install_dir=base_install_dir):
|
|
""" Download and install the given user app list at the specified
|
|
directories. The apps given must be archives that can be unpacked for
|
|
installation (e.g. portable distributions).
|
|
|
|
Parameters:
|
|
user_app_list (list): A list of dictionaries that contain app and
|
|
manifest information dictating what is to be installed via this
|
|
function.
|
|
download_dir (string): The path that intermediate downloaded files will
|
|
be kept at.
|
|
install_dir (string): The path that installed applications will be
|
|
unpacked to
|
|
|
|
Returns:
|
|
result (list): A list of dictionaries containing the install locations
|
|
of each app, e.g.
|
|
"""
|
|
|
|
base_downloads_dir = download_dir
|
|
base_install_dir = install_dir
|
|
|
|
# --------------------------------------------------------------------------
|
|
# This app list must always be installed, they provide the tools to install all
|
|
# other archives. Upon installation, we will collect the installation executable
|
|
# path and store them in global variables for the rest of the progam to use to
|
|
# unzip the files.
|
|
global internal_app_list
|
|
internal_app_list = []
|
|
|
|
internal_app_list.append({
|
|
"label": "7zip",
|
|
"manifests": [],
|
|
})
|
|
|
|
if is_windows or os.name == "nt":
|
|
version = "920"
|
|
internal_app_list[-1]["manifests"].append({ # Download the bootstrap 7zip, this can be unzipped using shutils
|
|
"download_checksum": "2a3afe19c180f8373fa02ff00254d5394fec0349f5804e0ad2f6067854ff28ac",
|
|
"download_url": f"https://www.7-zip.org/a/7za{version}.zip",
|
|
"version": version,
|
|
"executables": [
|
|
{
|
|
"path": "7za.exe",
|
|
"symlink": [],
|
|
"add_to_devenv_path": False,
|
|
"checksum": "c136b1467d669a725478a6110ebaaab3cb88a3d389dfa688e06173c066b76fcf"
|
|
}
|
|
],
|
|
"add_to_devenv_script": [],
|
|
})
|
|
|
|
version = ""
|
|
download_url = ""
|
|
download_checksum = ""
|
|
checksum = ""
|
|
exe_path = ""
|
|
|
|
if is_windows or os.name == "nt":
|
|
sevenz_version = "22.01"
|
|
zs_version = "1.5.5-R3"
|
|
version = f"{sevenz_version}-{zs_version}"
|
|
download_url = f"https://github.com/mcmilk/7-Zip-zstd/releases/download/v{sevenz_version}-v{zs_version}/7z{sevenz_version}-zstd-x64.exe"
|
|
download_checksum = "d542d78397bbed8e77c221f36cad461a0d83f1263b993a7048e81df40f403fb8"
|
|
checksum = "9d818c7a524ee2e8f81874c6067289c3eaa250887e25ee898937dfc1486bc066"
|
|
exe_path = "7z.exe"
|
|
|
|
else:
|
|
version = "2301"
|
|
download_url = f"https://www.7-zip.org/a/7z{version}-linux-x64.tar.xz"
|
|
download_checksum = "23babcab045b78016e443f862363e4ab63c77d75bc715c0b3463f6134cbcf318"
|
|
checksum = "c7f8769e2bc8df6bcbfba34571ee0340670a52dec824dbac844dd3b5bd1a69e1"
|
|
exe_path = "7zz"
|
|
|
|
internal_app_list[-1]["manifests"].append({ # Download proper 7zip, extract this exe with the bootstrap 7zip
|
|
"download_checksum": download_checksum,
|
|
"download_url": download_url,
|
|
"version": version,
|
|
"executables": [
|
|
{
|
|
"path": exe_path,
|
|
"symlink": [],
|
|
"add_to_devenv_path": True,
|
|
"checksum": checksum,
|
|
}
|
|
],
|
|
"add_to_devenv_script": [],
|
|
})
|
|
|
|
# ------------------------------------------------------------------------------
|
|
|
|
version = "1.5.2"
|
|
download_url = ""
|
|
download_checksum = ""
|
|
checksum = ""
|
|
exe_path = ""
|
|
|
|
if is_windows or os.name == 'nt':
|
|
download_url = f"https://github.com/facebook/zstd/releases/download/v{version}/zstd-v{version}-win64.zip"
|
|
download_checksum = "68897cd037ee5e44c6d36b4dbbd04f1cc4202f9037415a3251951b953a257a09"
|
|
checksum = "f14e78c0651851a670f508561d2c5d647da0ba08e6b73231f2e7539812bae311"
|
|
exe_path = "zstd.exe"
|
|
|
|
internal_app_list.append({
|
|
"label": "zstd",
|
|
"manifests": [
|
|
{
|
|
"download_checksum": download_checksum,
|
|
"download_url": download_url,
|
|
"version": version,
|
|
"executables": [
|
|
{
|
|
"path": exe_path,
|
|
"symlink": [],
|
|
"add_to_devenv_path": True,
|
|
"checksum": checksum,
|
|
},
|
|
],
|
|
"add_to_devenv_script": [],
|
|
},
|
|
],
|
|
})
|
|
|
|
# Run
|
|
# --------------------------------------------------------------------------
|
|
# Create the starting directories and install the internal app list (e.g.
|
|
# 7zip) which will be used to unzip-install the rest of the apps in the user
|
|
# app list.
|
|
#
|
|
# To do this without dependencies, we first download an old version of 7zip,
|
|
# version 9.20 which is distributed as a .zip file which Python can natively
|
|
# unzip.
|
|
#
|
|
# We then use the old version of 7zip and download a newer version of 7zip
|
|
# and extract it using the bootstrap-ed version. As of writing this, 7zip
|
|
# does not release a portable distribution of itself yet, instead what we do
|
|
# is download the installer and extract it ourselves using the bootstrap
|
|
# 7zip.
|
|
|
|
# Create the paths requested by the user
|
|
os.makedirs(download_dir, exist_ok=True)
|
|
os.makedirs(install_dir, exist_ok=True)
|
|
os.makedirs(pathlib.Path(install_dir, "Symlinks"), exist_ok=True)
|
|
|
|
for path in [download_dir, install_dir]:
|
|
if not os.path.isdir(path):
|
|
exit(f'Path "{path}" is not a directory, script can not proceed. Exiting.')
|
|
|
|
global devenv_script_buffer
|
|
if is_windows:
|
|
devenv_script_buffer = """@echo off
|
|
|
|
"""
|
|
|
|
# Validate all the manifests before starting
|
|
internal_app_validate_result = validate_app_list(internal_app_list)
|
|
user_app_validate_result = validate_app_list(user_app_list)
|
|
|
|
# Install apps
|
|
internal_apps = install_app_list(app_list=internal_app_list,
|
|
download_dir=download_dir,
|
|
install_dir=install_dir,
|
|
is_windows=is_windows)
|
|
|
|
user_apps = install_app_list(app_list=user_app_list,
|
|
download_dir=download_dir,
|
|
install_dir=install_dir,
|
|
is_windows=is_windows)
|
|
|
|
# Write the devenv script with environment variables
|
|
if is_windows:
|
|
devenv_script_buffer += "set devenver_root_backslash=%~dp0\n"
|
|
devenv_script_buffer += "set devenver_root=%devenver_root_backslash:~0,-1%\n"
|
|
devenv_script_buffer += "set devenver_root_backslash=\n"
|
|
|
|
devenv_script_buffer += "set path=%~dp0Symlinks;%PATH%\n"
|
|
devenv_script_buffer += "set path=%~dp0Scripts;%PATH%\n"
|
|
else:
|
|
devenv_script_buffer += f"export devenver_root=\"$( cd -- $( dirname -- \"${{BASH_SOURCE[0]}}\" ) &> /dev/null && pwd )\"\n"
|
|
devenv_script_buffer += f"PATH=\"$( cd -- $( dirname -- \"${{BASH_SOURCE[0]}}\" ) &> /dev/null && pwd )/Scripts\":$PATH\n"
|
|
devenv_script_buffer += f"PATH=\"$( cd -- $( dirname -- \"${{BASH_SOURCE[0]}}\" ) &> /dev/null && pwd )/Symlinks\":$PATH\n"
|
|
devenv_script_buffer += f"PATH=$(echo $PATH | awk -v RS=: -v ORS=: '/^\/mnt\/c/ {{next}} {{print}}')\n"
|
|
devenv_script_buffer += f"source $devenver_root/unix_fzf-completion.bash\n"
|
|
devenv_script_buffer += f"source $devenver_root/unix_fzf-key-bindings.bash\n"
|
|
|
|
devenv_script_name = f"{devenv_script_name}.bat" if is_windows else f"{devenv_script_name}.sh"
|
|
devenv_script_path = pathlib.Path(install_dir, devenv_script_name)
|
|
|
|
lprint(f"Writing script to augment the environment with installed applications: {devenv_script_path}")
|
|
devenv_script_path.write_text(devenv_script_buffer)
|
|
|
|
if not is_windows:
|
|
subprocess.run(args=["chmod", "+x", devenv_script_path])
|
|
|
|
# Merge the install dictionaries, this dictionary contains
|
|
# (app label) -> [array of installed versions]
|
|
result = internal_apps
|
|
for key, value in user_apps.items():
|
|
if key not in result:
|
|
result.update({key: value})
|
|
else:
|
|
result[key] += value
|
|
|
|
return result
|