diff --git a/swift-ci/README.md b/swift-ci/README.md index a245fa78..9f531d40 100644 --- a/swift-ci/README.md +++ b/swift-ci/README.md @@ -35,6 +35,114 @@ docker run \ /bin/bash -lc "cp -r /source/* /home/build-user/; ./swift/utils/build-script --preset buildbot_linux install_destdir=/home/build-user/swift-install installable_package=/home/build-user/swift-DEVELOPMENT-SNAPSHOT-$(date +'%F')-a.tar.gz" ``` +## Quick Start for Windows Development + +The Windows Docker image will setup an enviornment with Python, Visual Studio +Build Tools, and Git. It is setup to assume that the sources will be available +in `S:\SourceCache`. + +### Building and Tagging the image + +```powershell +cd master\windows\10.0.19044.1706 +docker image build --compress -t swift:swiftci . +``` + +### Running the image + +```powershell +docker run --rm -it -v %UserProfile%\data:S: swift:swiftci +``` + +### Building the Toolchain + +While we can build the toolchain in the containerized environment, the sources +are expected to reside on the host and is passed into the docker container as a +volume. The rest of automation expects that the Sources reside under a +directory with the name `SourceCache`. + +#### Clone the Sources + +```cmd +md %UserProfile%\data\SourceCache +cd %UserProfile%\data\SourceCache + +git clone -b stable/20220426 -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/llvm-project +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-cmark cmark +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-experimental-string-processing +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-corelibs-libdispatch +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-corelibs-foundation +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-corelibs-xctest +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-argument-parser +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-crypto +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-driver +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-llbuild llbuild +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-package-manager +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-system +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-tools-support-core +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/swift-installer-scripts +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/indexstore-db +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/apple/sourcekit-lsp +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/jpsim/Yams +git clone -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/compnerd/swift-build +git clone -t curl-7_77_0 -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/curl/curl +git clone -t v2.9.12 -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/gnome/libxml2 +git clone -t v1.2.11 -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/madler/zlib +git clone -b maint/maint-69 -c core.autocrlf=input -c core.symlink=true -c core.useBuiltinFSMonitor=false https://github.com/unicode-org/icu +``` + +#### Run Docker + +```cmd +docker run --rm -it -v %UserProfile%\data:S: swift:swiftci +``` + +#### Build the Toolchain + +This will build a full Swift toolchain distribution (llvm, clang, lld, lldb, +swift, swift-package-manger, SourceKit-LSP) and the Windows SDK (x86, x64, +ARM64). + +```cmd +S: +S:\SourceCache\swift\utils\build.cmd +``` + +#### Running Swift Tests + +The toolchain tests require some modifications to the path to find some of the +dependencies. The following will run the Swift test suite within the docker +container: + +```cmd +path S:\b\1\bin;S:\b\1\tools\swift\libdispatch-windows-x86_64-prefix\bin;%Path%;%ProgramFiles%\Git\usr\bin +ninja -C S:\b\1 check-swift +``` + +#### Using the Toolchain + +> **NOTE**: Running the test suite and using the toolchain near the production mode are mututally incompatible (due to the path changes). + +The build will generate a toolchain image that is roughly similar to the +installed version. The following can be run inside the docker container to use +the toolchain: + +```cmd +set SDKROOT=S:\Library\Developer\Platforms\Windows.platform\Developer\SDKs\Windows.sdk +path S:\Library\Developer\Platforms\Windows.platform\Developer\SDKs\Windows.sdk\usr\bin\x64;S:\Library\Developer\Toolchains\unknown-Asserts-development.xctoolchain\usr\bin;%Path% +``` + +Because the toolchain is built in the volume which is backed by the host, the +toolchain can be used on the host (assuming the dependencies such as Visual +Studio is installed and the module modules deployed). The adjusted paths below +should enable that: + +```cmd +set SDKROOT=%UserProfile%\data\Library\Developer\Platforms\Windows.platform\Developer\SDKs\Windows.sdk +path %UserProfile%\data\Library\Developer\Platforms\Windows.platform\Developer\SDKs\Windows.sdk\usr\bin\x64;%UserProfile%\data\Library\Developer\Toolchains\unknown-Asserts-development.xctoolchain\usr\bin;%Path% +``` + ## Contributions Contributions via pull requests are welcome and encouraged :) diff --git a/swift-ci/master/windows/10.0.19044.1706/Dockerfile b/swift-ci/master/windows/10.0.19044.1706/Dockerfile new file mode 100644 index 00000000..6889dfbc --- /dev/null +++ b/swift-ci/master/windows/10.0.19044.1706/Dockerfile @@ -0,0 +1,61 @@ +# escape=` + +# 10.0.19044.1706 is not yet published, use the 20H2 snapshot as that is +# believed to work better than the newer image. +FROM mcr.microsoft.com/windows/servercore:10.0.19042.1706 AS windows + +LABEL maintainer="Swift Infrastructure " +LABEL description="Docker Container for the Swift programming language" + +ARG GIT=https://github.com/git-for-windows/git/releases/download/v2.36.1.windows.1/Git-2.36.1-64-bit.exe +ARG PYTHON=https://www.python.org/ftp/python/3.10.4/python-3.10.4-amd64.exe + +# restore the default Windows shell for correct batch processing +SHELL ["cmd", "/S", "/C"] + +# Enable Developer Mode. +RUN reg add "HKEY_LOCAL_MACHINE\SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock" /t REG_DWORD /f /v "AllowDevelopmentWithoutDevLicense" /d "1" + +# Enable Long Paths +RUN reg add "HKEY_LOCAL_MACHINE\SYSTEM\CurrentControlSet\Control\FileSystem" /t REG_DWORD /f /v "LongPathsEnabled" /d "1" + +# Install Git. +# See: git-[version]-[bit].exe /SAVEINF=git.inf and /? +COPY git.inf . +RUN ` + curl -SLo git.exe %GIT% ` + && (start /w git.exe /SP- /VERYSILENT /SUPPRESSMSGBOXES /NOCANCEL /NORESTART /CLOSEAPPLICATIONS /FORCECLOSEAPPLICATIONS /LOADINF=git.inf ) ` + && del /q git.exe git.inf + +# Install Python. +# See: https://docs.python.org/3.10/using/windows.html +# FIXME: it appears that `PYTHONHOME` and `PYTHONPATH` are unset +COPY unattend.xml . +RUN ` + curl -SLo python.exe %PYTHON% ` + && (start /w python.exe /quiet ) ` + && del /q python.exe unattend.xml + +# Install Visual Studio Build Tools +RUN ` + curl -SLo vs_buildtools.exe https://aka.ms/vs/17/release/vs_buildtools.exe ` + && (start /w vs_buildtools.exe --quiet --wait --norestart --nocache ` + --add Microsoft.VisualStudio.Component.Windows11SDK.22000 ` + --add Microsoft.VisualStudio.Component.VC.ATL ` + --add Microsoft.VisualStudio.Component.VC.ATL.ARM ` + --add Microsoft.VisualStudio.Component.VC.ATL.ARM64 ` + --add Microsoft.VisualStudio.Component.VC.CMake.Project ` + --add Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` + --add Microsoft.VisualStudio.Component.VC.Tools.ARM64 ` + --add Microsoft.VisualStudio.Component.VC.Tools.ARM ` + || IF "%EXITCODE%"=="3010" EXIT 0) ` + && del /q vs_buildtools.exe +COPY InstallModules.cmd . +RUN InstallModules.cmd && del /q InstallModules.cmd + +# FIXME: we should use a non-Administrator user +# USER ContainerUser + +ENV PYTHONUTF8=1 +# Default to powershell +CMD ["powershell.exe", "-nologo", "-ExecutionPolicy", "Bypass"] diff --git a/swift-ci/master/windows/10.0.19044.1706/InstallModules.cmd b/swift-ci/master/windows/10.0.19044.1706/InstallModules.cmd new file mode 100644 index 00000000..cf0d74d1 --- /dev/null +++ b/swift-ci/master/windows/10.0.19044.1706/InstallModules.cmd @@ -0,0 +1,11 @@ +@echo off +setlocal +set vswhere=%ProgramFiles(x86)%\Microsoft Visual Studio\Installer\vswhere.exe +FOR /F "tokens=* usebackq" %%r IN (`"%vswhere%" -nologo -latest -all -prerelease -products * -property installationPath`) DO SET VsDevCmd=%%r\Common7\Tools\VsDevCmd.bat +CALL "%VsDevCmd%" -no_logo -host_arch=amd64 -arch=amd64 +mklink "%UniversalCRTSdkDir%\Include\%UCRTVersion%\ucrt\module.modulemap" S:\SourceCache\swift\stdlib\public\Platform\ucrt.modulemap +mklink "%UniversalCRTSdkDir%\Include\%UCRTVersion%\um\module.modulemap" S:\SourceCache\swift\stdlib\public\Platform\winsdk.modulemap +mklink "%VCToolsInstallDir%\include\module.modulemap" S:\SourceCache\swift\stdlib\public\Platform\visualc.modulemap +mklink "%VCToolsInstallDir%\include\visualc.apinotes" S:\SourceCache\swift\stdlib\public\Platform\visualc.apinotes +endlocal +@echo on diff --git a/swift-ci/master/windows/10.0.19044.1706/git.inf b/swift-ci/master/windows/10.0.19044.1706/git.inf new file mode 100644 index 00000000..914358af --- /dev/null +++ b/swift-ci/master/windows/10.0.19044.1706/git.inf @@ -0,0 +1,13 @@ +[Setup] +NoIcons=1 +Components=gitlfs +EditorOption=VIM +PathOption=Cmd +SSHOption=OpenSSH +TurtoiseOption=false +CURLOption=WinSSL +BashTerminalOption=ConHost +PerformanceTweaksFSCache=Enabled +EnableSymlinks=Enabled +EnablePseudoConsoltSupport=Disabled +EnableFSMonitor=Enabled diff --git a/swift-ci/master/windows/10.0.19044.1706/unattend.xml b/swift-ci/master/windows/10.0.19044.1706/unattend.xml new file mode 100644 index 00000000..f1a60607 --- /dev/null +++ b/swift-ci/master/windows/10.0.19044.1706/unattend.xml @@ -0,0 +1,18 @@ + + +