installer: Revamp

This commit is contained in:
2022-05-16 22:51:54 +10:00
parent 5ffc053f28
commit 4cd21634e0
17 changed files with 135 additions and 155 deletions
+105
View File
@@ -0,0 +1,105 @@
@echo off
setlocal
REM
REM Generate Standalone MSVC17 x86/x64 Toolchain from VS Installation
REM
REM Collects the necessary includes, binaries, DLLs and libs to around ~300mb on
REM disk from a VS installation that can invoke cl, and link as you typically
REM would after calling "vcvarsall.bat x64"
REM
REM This script generates the helper scripts within the generated toolchain.
REM cl_[x64|x86].bat: Invokes the compiler with required environment variables set
REM link_[x64|x86}.bat: Invokes the linker with required environment variables set
REM msvc_env_[x64|x86].bat: Setups the environment variables for the toolchain.
REM i.e. "call msvc_env_x64.bat" to set the current
REM session's environment variables for compiling with
REM the toolchain, akin to "vcvarsall.bat x64"
REM
REM Information about the necessary files and steps adapted from
REM Krouzu's Isolating MSVC19 https://gist.github.com/krouzu/19ddd4cb989264b11c7b3ba48c159be0
REM Paul Houle's Isolating MSVC14 Script
REM
REM For other Visual Studio versions, you may need to update the version numbers.
REM
REM Configuration (NOTE: Update arch to either, "x86", "x64" or "x86 x64" for both toolchains).
set arch=x86 x64
REM Source Directories (NOTE: Update the directories for your desired version)
set vs_version=2017
set msvc_version=14.16.27023
set win_sdk_version=10.0.17763.0
set vs_root=C:\Program Files (x86)\Microsoft Visual Studio\%vs_version%\Community
set vs1=%vs_root%\VC\Tools\MSVC\%msvc_version%
set vs2=C:\Program Files (x86)\Windows Kits\10\bin\%win_sdk_version%
set vs3=C:\Program Files (x86)\Windows Kits\10\Include\%win_sdk_version%
set vs4=C:\Program Files (x86)\Windows Kits\10\Lib\%win_sdk_version%
set dll1=%vs_root%\Common7\Tools\api-ms-win-*.dll
set dll2=C:\Windows\System32\*140*.dll
set dll3=C:\Windows\System32\ucrtbase*.dll
set dll4=C:\Windows\System32\VsGraphicsHelper.dll
REM Destination Directory
set dest=%~1
if "%dest%"=="" echo Usage: %~nx0 ^<output folder^>
if "%dest%"=="" goto :eof
if exist "%dest%" echo Directory "%dest%" already exists, exiting.
if exist "%dest%" goto :eof
REM Path/File Exist Check
for %%a in ("%vs1%" "%vs2%" "%dll1%" "%dll2%" "%dll3%" "%dll4%" "%vs3%" "%vs4%") do (
if not exist %%a echo Required file or path not found: %%a
if not exist %%a goto :eof
)
set copy_cmd=xcopy /I /S
REM MSVC Includes
%copy_cmd% "%vs1%\include" "%dest%\include"
%copy_cmd% "%vs3%\ucrt\*" "%dest%\include"
%copy_cmd% "%vs3%\shared\*" "%dest%\sdk\include"
%copy_cmd% "%vs3%\um\*" "%dest%\sdk\include"
REM MSVC Binaries/Libraries/DLLs
for %%a in (%arch%) do (
%copy_cmd% "%vs1%\bin\Hostx64\%%a" "%dest%\bin\%%a"
%copy_cmd% "%vs1%\lib\%%a" "%dest%\lib\%%a"
%copy_cmd% "%vs2%\%%a" "%dest%\sdk\bin\%%a"
%copy_cmd% "%vs4%\ucrt\%%a" "%dest%\lib\%%a"
%copy_cmd% "%vs4%\um\%%a" "%dest%\sdk\lib\%%a"
%copy_cmd% "%dll1%" "%dest%\sdk\bin\%%a"
%copy_cmd% "%dll2%" "%dest%\sdk\bin\%%a"
%copy_cmd% "%dll3%" "%dest%\sdk\bin\%%a"
%copy_cmd% "%dll4%" "%dest%\sdk\bin\%%a"
REM Generate Compiler/Linker Scripts
setlocal EnableDelayedExpansion
set msvc_env_script="%dest%\msvc_env_%%a.bat"
set cl_script="%dest%\cl_%%a.bat"
set link_script="%dest%\link_%%a.bat"
for %%b in (!msvc_env_script! !cl_script! !link_script!) do (
echo @echo off>> %%b
)
for %%b in (!cl_script! !link_script!) do (
echo setlocal>> %%b
)
for %%b in (!msvc_env_script! !cl_script! !link_script!) do (
echo set msvc_root=%%~dp0>> %%b
echo set include=%%msvc_root%%\include;%%msvc_root%%\sdk\include>> %%b
echo set lib=%%msvc_root%%\lib\%%a;%%msvc_root%%\sdk\lib\%%a>> %%b
echo set path=%%msvc_root%%\bin\%%a;%%path%%>> %%b
)
echo cl %%*>> !cl_script!
echo link %%*>> !link_script!
)
+104
View File
@@ -0,0 +1,104 @@
@echo off
setlocal
REM
REM Generate Standalone MSVC 2017/2019 x86/x64 Toolchain from VS Installation
REM
REM Collects the necessary includes, binaries, DLLs and libs to around ~300mb on
REM disk from a VS installation that can invoke cl, and link as you typically
REM would after calling "vcvarsall.bat x64"
REM
REM This script generates the helper scripts within the generated toolchain.
REM cl_[x64|x86].bat: Invokes the compiler with required environment variables set
REM link_[x64|x86].bat: Invokes the linker with required environment variables set
REM msvc_env_[x64|x86].bat: Setups the environment variables for the toolchain.
REM i.e. "call msvc_env_x64.bat" to set the current
REM session's environment variables for compiling with
REM the toolchain, akin to "vcvarsall.bat x64"
REM
REM Information about the necessary files and steps adapted from
REM Krouzu's Isolating MSVC19 https://gist.github.com/krouzu/19ddd4cb989264b11c7b3ba48c159be0
REM Paul Houle's Isolating MSVC14 Script
REM
REM For other Visual Studio versions, you may need to update the version numbers.
REM
REM Configuration (NOTE: Update arch to either, "x86", "x64" or "x86 x64" for both toolchains).
set arch=x86 x64
REM Source Directories (NOTE: Update the directories for your desired version)
set vs_version=2019
set msvc_version=14.28.29910
set win_sdk_version=10.0.19041.0
set vs_root=C:\Program Files (x86)\Microsoft Visual Studio\%vs_version%\Community
set vs1=%vs_root%\VC\Tools\MSVC\%msvc_version%
set vs2=C:\Program Files (x86)\Windows Kits\10\bin\%win_sdk_version%
set vs3=C:\Program Files (x86)\Windows Kits\10\Include\%win_sdk_version%
set vs4=C:\Program Files (x86)\Windows Kits\10\Lib\%win_sdk_version%
set dll1=%vs_root%\Common7\Tools\api-ms-win-*.dll
set dll2=C:\Windows\System32\*140*.dll
set dll3=C:\Windows\System32\ucrtbase*.dll
set dll4=C:\Windows\System32\VsGraphicsHelper.dll
REM Destination Directory
set dest=%~1
if "%dest%"=="" echo Usage: %~nx0 ^<output folder^>
if "%dest%"=="" goto :eof
if exist "%dest%" echo Directory "%dest%" already exists, exiting.
if exist "%dest%" goto :eof
REM Path/File Exist Check
for %%a in ("%vs1%" "%vs2%" "%dll1%" "%dll2%" "%dll3%" "%dll4%" "%vs3%" "%vs4%") do (
if not exist %%a echo Required file or path not found: %%a
if not exist %%a goto :eof
)
set copy_cmd=xcopy /I /S
REM MSVC Includes
%copy_cmd% "%vs1%\include" "%dest%\include"
%copy_cmd% "%vs3%\ucrt\*" "%dest%\include"
%copy_cmd% "%vs3%\shared\*" "%dest%\sdk\include"
%copy_cmd% "%vs3%\um\*" "%dest%\sdk\include"
REM MSVC Binaries/Libraries/DLLs
for %%a in (%arch%) do (
%copy_cmd% "%vs1%\bin\Hostx64\%%a" "%dest%\bin\%%a"
%copy_cmd% "%vs1%\lib\%%a" "%dest%\lib\%%a"
%copy_cmd% "%vs2%\%%a" "%dest%\sdk\bin\%%a"
%copy_cmd% "%vs4%\ucrt\%%a" "%dest%\lib\%%a"
%copy_cmd% "%vs4%\um\%%a" "%dest%\sdk\lib\%%a"
%copy_cmd% "%dll1%" "%dest%\sdk\bin\%%a"
%copy_cmd% "%dll2%" "%dest%\sdk\bin\%%a"
%copy_cmd% "%dll3%" "%dest%\sdk\bin\%%a"
%copy_cmd% "%dll4%" "%dest%\sdk\bin\%%a"
REM Generate Compiler/Linker Scripts
setlocal EnableDelayedExpansion
set msvc_env_script="%dest%\msvc_env_%%a.bat"
set cl_script="%dest%\cl_%%a.bat"
set link_script="%dest%\link_%%a.bat"
for %%b in (!msvc_env_script! !cl_script! !link_script!) do (
echo @echo off>> %%b
)
for %%b in (!cl_script! !link_script!) do (
echo setlocal>> %%b
)
for %%b in (!msvc_env_script! !cl_script! !link_script!) do (
echo set msvc_root=%%~dp0>> %%b
echo set include=%%msvc_root%%\include;%%msvc_root%%\sdk\include>> %%b
echo set lib=%%msvc_root%%\lib\%%a;%%msvc_root%%\sdk\lib\%%a>> %%b
echo set path=%%msvc_root%%\bin\%%a;%%path%%>> %%b
)
echo cl %%*>> !cl_script!
echo link %%*>> !link_script!
)
+9
View File
@@ -0,0 +1,9 @@
This downloads standalone 64-bit MSVC compiler, linker & other tools, also headers/libraries from Windows SDK into portable folder, without installing Visual Studio. Has bare minimum components - no UWP/Store/WindowsRT stuff, just files & tools for 64-bit native desktop app development.
Run `python.exe portable-msvc.py` and it will download output into `msvc` folder. By default it will download latest available MSVC & Windows SDK - currently v14.31 and v10.0.22000.0.
You can list available versions with `python.exe portable-msvc.py --show-versions` and then pass versions you want with `--msvc-version` and `--sdk-version` arguments.
To use cl.exe/link.exe from output folder, first run `setup.bat` - after that PATH/INCLUDE/LIB env variables will be setup to use all the tools as usual. You can also use clang-cl.exe with these includes & libraries.
To use clang-cl.exe without running setup.bat, pass extra `/winsysroot msvc` argument (msvc is folder name where output is stored).
+288
View File
@@ -0,0 +1,288 @@
#!/usr/bin/env python3
import io
import os
import sys
import json
import shutil
import hashlib
import zipfile
import tempfile
import argparse
import subprocess
import urllib.request
from pathlib import Path
OUTPUT = Path("msvc") # output folder
# other architectures may work or may not - not really tested
HOST = "x64" # or x86
TARGET = "x64" # or x86, arm, arm64
MANIFEST_URL = "https://aka.ms/vs/17/release/channel"
def download(url):
with urllib.request.urlopen(url) as res:
return res.read()
def download_progress(url, check, name, f):
data = io.BytesIO()
with urllib.request.urlopen(url) as res:
total = int(res.headers["Content-Length"])
size = 0
while True:
block = res.read(1<<20)
if not block:
break
f.write(block)
data.write(block)
size += len(block)
perc = size * 100 // total
print(f"\r{name} ... {perc}%", end="")
print()
data = data.getvalue()
digest = hashlib.sha256(data).hexdigest()
if check.lower() != digest:
exit(f"Hash mismatch for f{pkg}")
return data
# super crappy msi format parser just to find required .cab files
def get_msi_cabs(msi):
index = 0
while True:
index = msi.find(b".cab", index+4)
if index < 0:
return
yield msi[index-32:index+4].decode("ascii")
def first(items, cond):
return next(item for item in items if cond(item))
### parse command-line arguments
ap = argparse.ArgumentParser()
ap.add_argument("--show-versions", const=True, action="store_const", help="Show available MSVC and Windows SDK versions")
ap.add_argument("--accept-license", const=True, action="store_const", help="Automatically accept license")
ap.add_argument("--msvc-version", help="Get specific MSVC version")
ap.add_argument("--sdk-version", help="Get specific Windows SDK version")
args = ap.parse_args()
### get main manifest
manifest = json.loads(download(MANIFEST_URL))
### download VS manifest
vs = first(manifest["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Manifests.VisualStudio")
payload = vs["payloads"][0]["url"]
vsmanifest = json.loads(download(payload))
### find MSVC & WinSDK versions
packages = {}
for p in vsmanifest["packages"]:
packages.setdefault(p["id"].lower(), []).append(p)
msvc = {}
sdk = {}
for pid,p in packages.items():
if pid.startswith("Microsoft.VisualStudio.Component.VC.".lower()) and pid.endswith(".x86.x64".lower()):
pver = ".".join(pid.split(".")[4:6])
if pver[0].isnumeric():
msvc[pver] = pid
elif pid.startswith("Microsoft.VisualStudio.Component.Windows10SDK.".lower()) or \
pid.startswith("Microsoft.VisualStudio.Component.Windows11SDK.".lower()):
pver = pid.split(".")[-1]
if pver.isnumeric():
sdk[pver] = pid
if args.show_versions:
print("MSVC versions:", " ".join(sorted(msvc.keys())))
print("Windows SDK versions:", " ".join(sorted(sdk.keys())))
exit(0)
msvc_ver = args.msvc_version or max(sorted(msvc.keys()))
sdk_ver = args.sdk_version or max(sorted(sdk.keys()))
if msvc_ver in msvc:
msvc_pid = msvc[msvc_ver]
msvc_ver = ".".join(msvc_pid.split(".")[4:-2])
else:
exit(f"Unknown MSVC version: f{args.msvc_version}")
if sdk_ver in sdk:
sdk_pid = sdk[sdk_ver]
else:
exit(f"Unknown Windows SDK version: f{args.sdk_version}")
print(f"Downloading MSVC v{msvc_ver} and Windows SDK v{sdk_ver}")
### agree to license
tools = first(manifest["channelItems"], lambda x: x["id"] == "Microsoft.VisualStudio.Product.BuildTools")
resource = first(tools["localizedResources"], lambda x: x["language"] == "en-us")
license = resource["license"]
if not args.accept_license:
accept = input(f"Do you accept Visual Studio license at {license} [Y/N] ? ")
if not accept or accept[0].lower() != "y":
exit(0)
OUTPUT.mkdir(exist_ok=True)
total_download = 0
### download MSVC
msvc_packages = [
# MSVC binaries
f"microsoft.vc.{msvc_ver}.tools.host{HOST}.target{TARGET}.base",
f"microsoft.vc.{msvc_ver}.tools.host{HOST}.target{TARGET}.res.base",
# MSVC headers
f"microsoft.vc.{msvc_ver}.crt.headers.base",
# MSVC libs
f"microsoft.vc.{msvc_ver}.crt.{TARGET}.desktop.base",
f"microsoft.vc.{msvc_ver}.crt.{TARGET}.store.base",
# MSVC runtime source
f"microsoft.vc.{msvc_ver}.crt.source.base",
# ASAN
f"microsoft.vc.{msvc_ver}.asan.headers.base",
f"microsoft.vc.{msvc_ver}.asan.{TARGET}.base",
# MSVC redist
#f"microsoft.vc.{msvc_ver}.crt.redist.x64.base",
]
for pkg in msvc_packages:
p = first(packages[pkg], lambda p: p.get("language") in (None, "en-US"))
for payload in p["payloads"]:
with tempfile.TemporaryFile() as f:
data = download_progress(payload["url"], payload["sha256"], pkg, f)
total_download += len(data)
with zipfile.ZipFile(f) as z:
for name in z.namelist():
if name.startswith("Contents/"):
out = OUTPUT / Path(name).relative_to("Contents")
out.parent.mkdir(parents=True, exist_ok=True)
out.write_bytes(z.read(name))
### download Windows SDK
sdk_packages = [
# Windows SDK tools (like rc.exe & mt.exe)
f"Windows SDK for Windows Store Apps Tools-x86_en-us.msi",
# Windows SDK headers
f"Windows SDK for Windows Store Apps Headers-x86_en-us.msi",
# Windows SDK libs
f"Windows SDK for Windows Store Apps Libs-x86_en-us.msi",
f"Windows SDK Desktop Libs {TARGET}-x86_en-us.msi",
# CRT headers & libs
f"Universal CRT Headers Libraries and Sources-x86_en-us.msi",
# CRT redist
#"Universal CRT Redistributable-x86_en-us.msi",
]
with tempfile.TemporaryDirectory() as d:
dst = Path(d)
sdk_pkg = packages[sdk_pid][0]
sdk_pkg = packages[first(sdk_pkg["dependencies"], lambda x: True).lower()][0]
msi = []
cabs = []
# download msi files
for pkg in sdk_packages:
payload = first(sdk_pkg["payloads"], lambda p: p["fileName"] == f"Installers\\{pkg}")
msi.append(dst / pkg)
with open(dst / pkg, "wb") as f:
data = download_progress(payload["url"], payload["sha256"], pkg, f)
total_download += len(data)
cabs += list(get_msi_cabs(data))
# download .cab files
for pkg in cabs:
payload = first(sdk_pkg["payloads"], lambda p: p["fileName"] == f"Installers\\{pkg}")
with open(dst / pkg, "wb") as f:
download_progress(payload["url"], payload["sha256"], pkg, f)
print("Unpacking msi files...")
# run msi installers
for m in msi:
subprocess.check_call(["msiexec.exe", "/a", m, "/quiet", "/qn", f"TARGETDIR={OUTPUT.resolve()}"])
### versions
msvcv = list((OUTPUT / "VC/Tools/MSVC").glob("*"))[0].name
sdkv = list((OUTPUT / "Windows Kits/10/bin").glob("*"))[0].name
# place debug CRT runtime into MSVC folder (not what real Visual Studio installer does... but is reasonable)
dst = str(OUTPUT / "VC/Tools/MSVC" / msvcv / f"bin/Host{HOST}/{TARGET}")
pkg = "microsoft.visualcpp.runtimedebug.14"
dbg = packages[pkg][0]
payload = first(dbg["payloads"], lambda p: p["fileName"] == "cab1.cab")
try:
with tempfile.TemporaryFile(suffix=".cab", delete=False) as f:
data = download_progress(payload["url"], payload["sha256"], pkg, f)
total_download += len(data)
subprocess.check_call(["expand.exe", f.name, "-F:*", dst], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
finally:
os.unlink(f.name)
### cleanup
shutil.rmtree(OUTPUT / "Common7", ignore_errors=True)
for f in ["Auxiliary", f"lib/{TARGET}/store", f"lib/{TARGET}/uwp"]:
shutil.rmtree(OUTPUT / "VC/Tools/MSVC" / msvcv / f)
for f in OUTPUT.glob("*.msi"):
f.unlink()
for f in ["Catalogs", "DesignTime", f"bin/{sdkv}/chpe", f"Lib/{sdkv}/ucrt_enclave"]:
shutil.rmtree(OUTPUT / "Windows Kits/10" / f, ignore_errors=True)
for arch in ["x86", "x64", "arm", "arm64"]:
if arch != TARGET:
shutil.rmtree(OUTPUT / "VC/Tools/MSVC" / msvcv / f"bin/Host{arch}", ignore_errors=True)
shutil.rmtree(OUTPUT / "Windows Kits/10/bin" / sdkv / arch)
shutil.rmtree(OUTPUT / "Windows Kits/10/Lib" / sdkv / "ucrt" / arch)
shutil.rmtree(OUTPUT / "Windows Kits/10/Lib" / sdkv / "um" / arch)
### setup.bat
SETUP = f"""@echo off
set ROOT=%~dp0
set MSVC_VERSION={msvcv}
set MSVC_HOST=Host{HOST}
set MSVC_ARCH={TARGET}
set SDK_VERSION={sdkv}
set SDK_ARCH={TARGET}
set MSVC_ROOT=%ROOT%VC\\Tools\\MSVC\\%MSVC_VERSION%
set SDK_INCLUDE=%ROOT%Windows Kits\\10\\Include\\%SDK_VERSION%
set SDK_LIBS=%ROOT%Windows Kits\\10\\Lib\\%SDK_VERSION%
set VCToolsInstallDir=%MSVC_ROOT%\\
set PATH=%MSVC_ROOT%\\bin\\%MSVC_HOST%\\%MSVC_ARCH%;%ROOT%Windows Kits\\10\\bin\\%SDK_VERSION%\\%SDK_ARCH%;%ROOT%Windows Kits\\10\\bin\\%SDK_VERSION%\\%SDK_ARCH%\\ucrt;%PATH%
set INCLUDE=%MSVC_ROOT%\\include;%SDK_INCLUDE%\\ucrt;%SDK_INCLUDE%\\shared;%SDK_INCLUDE%\\um;%SDK_INCLUDE%\\winrt;%SDK_INCLUDE%\\cppwinrt
set LIB=%MSVC_ROOT%\\lib\\%MSVC_ARCH%;%SDK_LIBS%\\ucrt\\%SDK_ARCH%;%SDK_LIBS%\\um\\%SDK_ARCH%
"""
with open(OUTPUT / "setup.bat", "w") as f:
print(SETUP, file=f)
print(f"Total downloaded: {total_download>>20} MB")
print("Done!")