From fbfd7a9839e63f636b06ac211d31c265a33b9374 Mon Sep 17 00:00:00 2001 From: Samruddhi Khandale Date: Wed, 1 Jun 2022 17:27:28 -0700 Subject: [PATCH] devcontainer changes --- .github/workflows/push-dev.yml | 99 ++++++++++++++++++++++ .github/workflows/smoke-python.yaml | 4 +- LICENSE | 21 +++++ build/README.md | 74 ++++++++-------- build/assets/release-notes-header.md | 4 +- build/config.json | 31 +++---- build/src/prep.js | 79 +---------------- build/src/push.js | 38 +++------ build/src/utils/config.js | 17 +--- build/src/utils/image-content-extractor.js | 2 +- build/vscdc | 68 +++------------ 11 files changed, 204 insertions(+), 233 deletions(-) create mode 100644 .github/workflows/push-dev.yml create mode 100644 LICENSE diff --git a/.github/workflows/push-dev.yml b/.github/workflows/push-dev.yml new file mode 100644 index 000000000..1cf875cc6 --- /dev/null +++ b/.github/workflows/push-dev.yml @@ -0,0 +1,99 @@ +name: Build and push "dev" images + +on: + workflow_dispatch: + # schedule: + # - cron: '0 14 * * MON' + +jobs: + build-and-push: + name: Build and push + if: "!contains(github.event.head_commit.message, 'Automated update') && !contains(github.event.head_commit.message, 'CI ignore')" + strategy: + matrix: + page: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + page-total: [10] + fail-fast: true + runs-on: ubuntu-latest + steps: + - name: Free more space + id: free_space + run: | + set -e + # Ensure enough space is available for build + sudo apt-get autoremove -y + sudo apt-get clean -y + sudo rm -rf /usr/share/dotnet + + - name: Checkout + id: checkout + uses: actions/checkout@v1 + + - name: Azure CLI login + id: az_login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZ_ACR_CREDS }} + + - name: Build and push dev tags + id: build_and_push + run: | + set -e + + # ACR login + ACR_REGISTRY_NAME=$(echo ${{ secrets.REGISTRY }} | grep -oP '(.+)(?=\.azurecr\.io)') + az acr login --name $ACR_REGISTRY_NAME + + # Build and push dev images + yarn install + GIT_BRANCH=$(echo "${{ github.ref }}" | grep -oP 'refs/(heads|tags)/\K(.+)') + if [ "$GIT_BRANCH" == "" ]; then + GIT_BRANCH=main + fi + build/vscdc push --page ${{ matrix.page }} \ + --pageTotal ${{ matrix.page-total }} \ + --release $GIT_BRANCH \ + --github-repo ${{ github.repository }} \ + --registry ${{ secrets.REGISTRY }} \ + --registry-path ${{ secrets.REGISTRY_BASE_PATH }} \ + --stub-registry ${{ secrets.STUB_REGISTRY }} \ + --stub-registry-path ${{ secrets.STUB_REGISTRY_BASE_PATH }} + + package: + name: Package + if: "!contains(github.event.head_commit.message, 'Automated update') && !contains(github.event.head_commit.message, 'CI ignore')" + needs: [build-and-push] + runs-on: ubuntu-latest + steps: + - name: Checkout + id: checkout + uses: actions/checkout@v1 + + - name: Package + id: package + run: | + set -e + yarn install + GIT_BRANCH=$(echo "${{ github.ref }}" | grep -oP 'refs/(heads|tags)/\K(.+)') + if [ "$GIT_BRANCH" == "" ]; then + GIT_BRANCH=main + fi + build/vscdc pack --prep-and-package-only \ + --release $GIT_BRANCH \ + --github-repo ${{ github.repository }} \ + --registry ${{ secrets.REGISTRY }} \ + --registry-path ${{ secrets.REGISTRY_BASE_PATH }} \ + --stub-registry ${{ secrets.STUB_REGISTRY }} \ + --stub-registry-path ${{ secrets.STUB_REGISTRY_BASE_PATH }} + + # Set an output with the resulting package name for upload + PKG_PREFIX=$(node -p "require('./package.json').name") + PKG_NAME=$PKG_PREFIX-${{ github.sha }}.tgz + mv ./$PKG_PREFIX-*.tgz ./$PKG_NAME + echo "::set-output name=package_name::$PKG_NAME" + + - name: Upload package + uses: actions/upload-artifact@v1.0.0 + with: + name: ${{ steps.package.outputs.package_name }} + path: ./${{ steps.package.outputs.package_name }} \ No newline at end of file diff --git a/.github/workflows/smoke-python.yaml b/.github/workflows/smoke-python.yaml index 1ce25db18..4c1c866db 100644 --- a/.github/workflows/smoke-python.yaml +++ b/.github/workflows/smoke-python.yaml @@ -6,12 +6,12 @@ on: push: branches: [main] paths: - - 'containers/python/**' + - 'src/python/**' pull_request: branches: - main paths: - - 'containers/python/**' + - 'src/python/**' jobs: smoke-test: name: Smoke test diff --git a/LICENSE b/LICENSE new file mode 100644 index 000000000..7208551df --- /dev/null +++ b/LICENSE @@ -0,0 +1,21 @@ + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE \ No newline at end of file diff --git a/build/README.md b/build/README.md index 02d1eea30..7b1173b4a 100644 --- a/build/README.md +++ b/build/README.md @@ -1,4 +1,4 @@ -# Build and image generation for vscode-dev-containers +# Build and image generation for devcontainers This folder contains scripts to build and push images into the Microsoft Container Registry (MCR) from this repository, generate or modify any associated content to use the built image, track dependencies, and create an npm package with the result that is shipped in the VS Code Remote - Containers and Codespaces extension. @@ -49,19 +49,19 @@ Once you have your build configuration setup, you can use the `vscdc` CLI to tes 1. First, build the image(s) using the CLI as follows: ```bash - build/vscdc push --no-push --registry mcr.microsoft.com --registry-path vscode/devcontainers --release main + build/vscdc push --no-push --registry mcr.microsoft.com --registry-path devcontainers --release main ``` 2. Use the Docker CLI to verify all of the expected images and tags and have the right contents: ```bash - docker run -it --init --privileged --rm mcr.microsoft.com/vscode/devcontainers/:dev- bash + docker run -it --init --privileged --rm mcr.microsoft.com/devcontainers/:dev- bash ``` 3. Finally, test cgmanifest/markdown generation by running: ```bash - build/vscdc cg --registry mcr.microsoft.com --registry-path vscode/devcontainers --release main + build/vscdc cg --registry mcr.microsoft.com --registry-path devcontainers --release main ``` Once you're happy with the result, you can also verify that the `devcontainer.json` and the associated concent that will be generated for your definition is correct. @@ -72,7 +72,7 @@ Once you're happy with the result, you can also verify that the `devcontainer.js build/vscdc pack --prep-and-package-only --release main ``` - A new file called `vscode-dev-containers--dev.tgz` should be in the root of the repository once this is done. + A new file called `devcontainers--dev.tgz` should be in the root of the repository once this is done. 2. Unzip generated the `tgz` somewhere in your filesystem. @@ -97,7 +97,7 @@ In your `Dockerfile`: ```Dockerfile ARG VARIANT=3 -FROM mcr.microsoft.com/vscode/devcontainers/python:${VARIANT} +FROM mcr.microsoft.com/devcontainers/python:${VARIANT} ``` In `devcontainer.json`: @@ -112,7 +112,7 @@ In `devcontainer.json`: ## Using the script library -The `/script-library` folder in this repository contains a number of scripts to install tools or configure container contents. Of particular note is `common-debain.sh` that should generally be run in any definition that does not extend from an existing `mcr.microsoft.com/vscode/devcontainers` image. +The `/script-library` folder in this repository contains a number of scripts to install tools or configure container contents. Of particular note is `common-debain.sh` that should generally be run in any definition that does not extend from an existing `mcr.microsoft.com/devcontainers` image. Since Dockerfiles can only COPY files relative to the Dockerfile itself, we cannot easily copy contents from a folder several levels up. This step could be scripted, but this becomes cumbersome when creating the definitions to begin with. Instead, the scripts can be added into a `.devcontainer/library-scripts` folder in the definition. A GitHub Actions workflow will automatically update files with the same name in this folder whenever something in `/script-library` is updated. @@ -139,7 +139,7 @@ To add one: Or if `common-debian.sh` was already run in your upstream image, you can copy it directly to the correct spot: ```Dockerfile - COPY library-scripts/meta.env /usr/local/etc/vscode-dev-containers/ + COPY library-scripts/meta.env /usr/local/etc/devcontainers/ ``` The build system will then automatically populate the file with the correct contents on build. @@ -171,14 +171,14 @@ The **`build.rootDistro`** property can be `debian`, `alpine`, or `redhat` curre The **`build.latest`** and **`build.tags`** properties affect how tags are applied. For example, here is how several dev container folders map: ```text -debian => mcr.microsoft.com/vscode/devcontainers/base:debian -alpine => mcr.microsoft.com/vscode/devcontainers/base:alpine -ubnutu => mcr.microsoft.com/vscode/devcontainers/base:ubuntu +debian => mcr.microsoft.com/devcontainers/base:debian +alpine => mcr.microsoft.com/devcontainers/base:alpine +ubnutu => mcr.microsoft.com/devcontainers/base:ubuntu ``` This results in just one "repository" in the registry much like you would see for other images in Docker Hub. -- mcr.microsoft.com/vscode/devcontainers/base +- mcr.microsoft.com/devcontainers/base The package version is then automatically added to these various tags in the `${VERSION}` location for an item in the `tags` property array as a part of the release. For example, release 0.40.0 would result in: @@ -245,12 +245,12 @@ FROM python:${VARIANT} This configuration would cause separate image variants, each with a different `VARIANT` build argument value passed in, that are then tagged as follows: -- mcr.microsoft.com/vscode/devcontainers/python:3 -- mcr.microsoft.com/vscode/devcontainers/python:3.6 -- mcr.microsoft.com/vscode/devcontainers/python:3.7 -- mcr.microsoft.com/vscode/devcontainers/python:3.8 +- mcr.microsoft.com/devcontainers/python:3 +- mcr.microsoft.com/devcontainers/python:3.6 +- mcr.microsoft.com/devcontainers/python:3.7 +- mcr.microsoft.com/devcontainers/python:3.8 -In addition `mcr.microsoft.com/vscode/devcontainers/python` would point to `mcr.microsoft.com/vscode/devcontainers/python:3` since it is the first in the list. +In addition `mcr.microsoft.com/devcontainers/python` would point to `mcr.microsoft.com/devcontainers/python:3` since it is the first in the list. #### The `build.variantTags` property @@ -286,11 +286,11 @@ For example: In this case, the image built for the `bullseye` variant will be tagged as follows: -- mcr.microsoft.com/vscode/devcontaienrs/base:latest -- mcr.microsoft.com/vscode/devcontaienrs/base:bullseye -- mcr.microsoft.com/vscode/devcontaienrs/base:debian -- mcr.microsoft.com/vscode/devcontaienrs/base:debian-11 -- mcr.microsoft.com/vscode/devcontaienrs/base:debian11 +- mcr.microsoft.com/devcontaienrs/base:latest +- mcr.microsoft.com/devcontaienrs/base:bullseye +- mcr.microsoft.com/devcontaienrs/base:debian +- mcr.microsoft.com/devcontaienrs/base:debian-11 +- mcr.microsoft.com/devcontaienrs/base:debian11 #### The `build.variantBuildArgs` property In some cases, you may need to vary build arguments in the definition's `base.Dockerfile` by variant (beyond the `VARIANT` build arg itself). This can be done using the `build.variantBuildArgs` property. For example, consider the following: @@ -522,11 +522,11 @@ Since the same dependencies can be in more than one definition, default settings ## Build process details and background -Currently the vscode-dev-containers repo contains pure Dockerfiles that need to be built when used. While this works well from the perspective of providing samples, some of the images install a significant number of runtimes or tools which can take a long time to build. +Currently the devcontainers repo contains pure Dockerfiles that need to be built when used. While this works well from the perspective of providing samples, some of the images install a significant number of runtimes or tools which can take a long time to build. We can resolve this by pre-building some of these images, but in-so-doing we want to make sure we: -1. Ensure vscode-dev-containers continues to be a good source of samples +1. Ensure devcontainers continues to be a good source of samples 2. Improve the performance using the most popular (or in some cases slowest to build) container images 3. Make it easy for users to add additional software to the images 4. Make it easy for contributors to build, customize, and contribute new definitions @@ -553,7 +553,7 @@ In the unlikely event a break fix needed to be deployed and not tagged latest, w This has a few advantages: -1. Tags are generated for each version of the repository that is cut. This allows us to refer directly to the exact version of the source code used to build the container image to avoid confusion. e.g. `https://github.com/microsoft/vscode-dev-containers/tree/v0.35.0/containers/javascript-node-8` +1. Tags are generated for each version of the repository that is cut. This allows us to refer directly to the exact version of the source code used to build the container image to avoid confusion. e.g. `https://github.com/microsoft/devcontainers/tree/v0.35.0/containers/javascript-node-8` 2. Similarly, as containers are deprecated and removed from the repository, you can still refer back to the container source and README. 3. Upstream changes that break existing images can be handled as needed. 4. Developers can opt to use the image tag 0.35 to get the latest break fix version if desired or 0 to always get the latest non-breaking update. @@ -576,7 +576,7 @@ Since the images would continue to exist after this point and the source code is When a release is cut, there are a few things that need to happen. One is obviously releasing the appropriate image. However, to continue to help customers understand how to customize their images, we would want to reference a user modifiable "stub" Dockerfile instead of an image directly. This also is important to help deal with shortcomings and workarounds that require something like specifying a build argument. For example: ```Dockerfile -FROM mcr.microsoft.com/vscode/devcontainer/javascript-node:0-10 +FROM mcr.microsoft.com/devcontainer/javascript-node:0-10 # ** [Optional] Uncomment this section to install additional packages. ** # @@ -603,9 +603,9 @@ Consequently, this user stub Dockerfile needs to be versioned with the `devconta The `definition-manifest.json` file dictates how the build process should behave as [dscribed above](#setting-up-a-container-to-be-built). In this case, `devcontainer.json` points to `base.Dockerfile`, but this is the Dockerfile used to generate the actual image rather than the stub Dockerfile. The stub that references the image is in `base.Dockerfile`. To make things easy, we can also automatically generate this stub at release time if only a Dockerfile is present. If no `base.Dockerfile` is found, the build process falls back to using `Dockerfile`. -Testing, then, is as simple as it is now - open the folder in `vscode-dev-containers` in a container and edit / test as required. Anyone simply copying the folder contents then gets a fully working version of the container even if in-flight and there is no image for it yet. +Testing, then, is as simple as it is now - open the folder in `devcontainers` in a container and edit / test as required. Anyone simply copying the folder contents then gets a fully working version of the container even if in-flight and there is no image for it yet. -In the vscode-dev-containers repo itself, the `FROM` statement in `Dockerfile` would always point to `latest` or `dev` since it what is in main may not have even been released yet. This would get dynamically updated as a part of the release process - which we will cover next. +In the devcontainers repo itself, the `FROM` statement in `Dockerfile` would always point to `latest` or `dev` since it what is in main may not have even been released yet. This would get dynamically updated as a part of the release process - which we will cover next. ```Dockerfile FROM mcr.microsoft.com/vs/devcontainer/javascript-node:dev-10 @@ -623,7 +623,7 @@ When a release is cut, this SHA is generated and the source code for the related #### Release process -When a release is cut, the contents of vscode-dev-containers repo are staged. The build process then does the following for the appropriate dev containers: +When a release is cut, the contents of devcontainers repo are staged. The build process then does the following for the appropriate dev containers: 1. Build an image using the `base.Dockerfile` and push it to a container registry with the appropriate version tags. If no `base.Dockerfile` is found, `Dockerfile` is used instead. If the `variants` property is set, one image is built per variant, with the variant value being passed in as the `VARIANT` build argument. @@ -633,22 +633,22 @@ When a release is cut, the contents of vscode-dev-containers repo are staged. Th ```Dockerfile # For information on the contents of the image referenced below, see the Dockerfile at - # https://github.com/microsoft/vscode-dev-containers/tree/v0.35.0/containers/javascript-node-10/.devcontainer/base.Dockerfile - FROM mcr.microsoft.com/vscode/devcontainer/javascript-node:0-10 + # https://github.com/microsoft/devcontainers/tree/v0.35.0/containers/javascript-node-10/.devcontainer/base.Dockerfile + FROM mcr.microsoft.com/devcontainer/javascript-node:0-10 ``` This also works when the `VARIANT` ARG is used. The MAJOR part of the release version is placed in front of the argument in the FROM statement: ```Dockerfile ARG VARIANT="3" - FROM mcr.microsoft.com/vscode/devcontainer/python:0-${VARIANT} + FROM mcr.microsoft.com/devcontainer/python:0-${VARIANT} ``` 4. `devcontainer.json` is updated to point to `Dockerfile` instead of `base.Dockerfile` (if required) and a comment is added that points to the definition in this repository (along with its associated README for this specific version). ```json // For format details, see https://aka.ms/vscode-remote/devcontainer.json or the definition README at - // https://github.com/microsoft/vscode-dev-containers/tree/v0.35.0/containers/javascript-node-10 + // https://github.com/microsoft/devcontainers/tree/v0.35.0/containers/javascript-node-10 { "name": "Node.js 10", "dockerFile": "Dockerfile", @@ -660,19 +660,19 @@ When a release is cut, the contents of vscode-dev-containers repo are staged. Th After everything builds successfully, the packaging process kicks off and performs the following: -1. Runs through all Dockerfiles in the `containers` folder and makes sure any references to `mcr.microsoft.com/vscode/devcontainers` in other non-built dockerfiles reference the MAJOR version as described in step 3 above. +1. Runs through all Dockerfiles in the `containers` folder and makes sure any references to `mcr.microsoft.com/devcontainers` in other non-built dockerfiles reference the MAJOR version as described in step 3 above. 2. Runs through all Dockerfiles and looks for [common script](#common-scripts) references and updates the URL to the tagged version and adds the expected SHA as another arg. The result is that sections of the Dockerfile that look like this: ```Dockerfile - ARG COMMON_SCRIPT_SOURCE="https://raw.githubusercontent.com/microsoft/vscode-dev-containers/main/script-library/common-debian.sh" + ARG COMMON_SCRIPT_SOURCE="https://raw.githubusercontent.com/microsoft/devcontainers/main/script-library/common-debian.sh" ARG COMMON_SCRIPT_SHA="dev-mode" ``` are transformed into this: ```Dockerfile - ARG COMMON_SCRIPT_SOURCE="https://raw.githubusercontent.com/microsoft/vscode-dev-containers/v0.112.0/script-library/common-debian.sh" + ARG COMMON_SCRIPT_SOURCE="https://raw.githubusercontent.com/microsoft/devcontainers/v0.112.0/script-library/common-debian.sh" ARG COMMON_SCRIPT_SHA="28e3d552a08e0d82935ad7335837f354809bec9856a3e0c2855f17bfe3a19523" ``` @@ -713,7 +713,7 @@ docker buildx build --build-arg VARIANT=6.0.100-bullseye-slim-arm64v8 --platform Once the build is complete, run the image using the below example. Note that the dotnet directory is mounted. The dotnet directory includes test scripts which will be used in the subsequent steps. ```bash -docker run -v $REPODIR/vscode-dev-containers/containers/dotnet/:/workspace --platform linux/arm64 -it dotnet-arm64 bash +docker run -v $REPODIR/devcontainers/containers/dotnet/:/workspace --platform linux/arm64 -it dotnet-arm64 bash ``` Once in the running container, verify that the architecture is ARM64 by running the command: diff --git a/build/assets/release-notes-header.md b/build/assets/release-notes-header.md index 50c456c55..35492f095 100644 --- a/build/assets/release-notes-header.md +++ b/build/assets/release-notes-header.md @@ -1,11 +1,11 @@ -# [{{definition}}](https://github.com/{{repository}}/tree/main/containers/{{definition}}) +# [{{definition}}](https://github.com/{{repository}}/tree/main/src/{{definition}}) {{#if annotation}} {{{annotation}}} {{/if}} **Image version:** {{version}} -**Source release/branch:** [{{release}}](https://github.com/{{repository}}/tree/{{release}}/containers/{{definition}}) +**Source release/branch:** [{{release}}](https://github.com/{{repository}}/tree/{{release}}/src/{{definition}}) {{#if hasVariants}} **Definition variations:** diff --git a/build/config.json b/build/config.json index 1bebe8ceb..28d344997 100644 --- a/build/config.json +++ b/build/config.json @@ -1,37 +1,26 @@ { "containerRegistry": "devcon.azurecr.io", - "containerRegistryPath": "public/vscode/devcontainers", + "containerRegistryPath": "public/devcontainers", "stubRegistry": "mcr.microsoft.com", - "stubRegistryPath": "vscode/devcontainers", - - "githubRepoName": "microsoft/vscode-dev-containers", - "containersPathInRepo": "containers", + "stubRegistryPath": "devcontainers", + "githubRepoName": "devcontainers/images", "historyFolderName": "history", - "repoContainersToBuildPath": "repository-containers/images", - "scriptLibraryPathInRepo": "script-library", - "scriptLibraryFolderNameInDefinition": "library-scripts", - "historyUrlPrefix": "https://github.com/microsoft/vscode-dev-containers/tree/main/containers/", - "repositoryUrl": "https://github.com/microsoft/vscode-dev-containers/", + "historyUrlPrefix": "https://github.com/devcontainers/images/tree/main/src/", + "repositoryUrl": "https://github.com/devcontainers/images", "imageLabelPrefix": "com.visualstudio.code.devcontainers", - "definitionBuildConfigFile": "definition-manifest.json", - + "definitionBuildConfigFile": "manifest.json", "devContainerJsonPreamble": "For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:\n// ", "dockerFilePreamble": "See here for image contents: ", "filesToStage": [ - "+(containers|repository-containers)/**/!(test-project|history)/*", - "script-library/**/*", - "LICENSE", - "package.json", - "yarn.lock", - ".npmignore", - "devcontainer-collection.json" + "+(src)/**/!(test-project|history)/*", + "LICENSE" ], "needsDedicatedPage": [ - "codespaces-linux", + "codespaces", "php", - "python-3" + "python" ], "flattenBaseImage": [], diff --git a/build/src/prep.js b/build/src/prep.js index 76ac635f6..6ab16445c 100644 --- a/build/src/prep.js +++ b/build/src/prep.js @@ -21,8 +21,6 @@ const stubPromises = { } const dockerFilePreamble = configUtils.getConfig('dockerFilePreamble'); -const scriptLibraryPathInRepo = configUtils.getConfig('scriptLibraryPathInRepo'); -const scriptLibraryFolderNameInDefinition = configUtils.getConfig('scriptLibraryFolderNameInDefinition'); const historyUrlPrefix = configUtils.getConfig('historyUrlPrefix'); const repositoryUrl = configUtils.getConfig('repositoryUrl'); @@ -54,9 +52,6 @@ async function prepDockerFile(devContainerDockerfilePath, definitionId, repo, re } }; - // Copy any scripts from the script library, add meta.env into the appropriate definition specific folder - await copyLibraryScriptsForDefinition(devContainerJsonPath, isForBuild, prepResult.meta); - if (isForBuild) { // If building, update FROM to target registry and version if definition has a parent const parentTag = configUtils.getParentTagForVersion(definitionId, version, registry, registryPath, variant); @@ -88,7 +83,7 @@ async function prepDockerFile(devContainerDockerfilePath, definitionId, repo, re // Otherwise update any Dockerfiles that refer to an un-versioned tag of another dev container // to the MAJOR version from this release. const expectedRegistry = configUtils.getConfig('stubRegistry', 'mcr.microsoft.com'); - const expectedRegistryPath = configUtils.getConfig('stubRegistryPath', 'vscode/devcontainers'); + const expectedRegistryPath = configUtils.getConfig('stubRegistryPath', 'devcontainers'); const fromCaptureGroups = new RegExp(`FROM\\s+(${expectedRegistry}/${expectedRegistryPath}/.+:.+)`).exec(devContainerDockerfileRaw); if (fromCaptureGroups && fromCaptureGroups.length > 0) { const fromDefinitionTag = configUtils.getUpdatedTag( @@ -127,7 +122,7 @@ async function updateStub(dotDevContainerPath, definitionId, repo, release, base async function processStub(userDockerFile, definitionId, repo, release, baseDockerFileExists, registry, registryPath) { const devContainerImageVersion = configUtils.majorFromRelease(release, definitionId); const relativePath = configUtils.getDefinitionPath(definitionId, true); - let fromSection = `# ${dockerFilePreamble}https://github.com/${repo}/tree/${release}/${relativePath}/.devcontainer/${baseDockerFileExists ? 'base.' : ''}Dockerfile\n\n`; + let fromSection = `# ${dockerFilePreamble}https://github.com/${repo}/tree/${release}/${relativePath}/Dockerfile\n\n`; // The VARIANT arg allows this value to be set from devcontainer.json, handle it if found if (/ARG\s+VARIANT\s*=/.exec(userDockerFile) !== null) { const variant = configUtils.getVariants(definitionId)[0]; @@ -151,7 +146,7 @@ async function updateConfigForRelease(definitionId, repo, release, registry, reg console.log(`(*) Making version specific updates to ${definitionId}...`); const definitionPath = configUtils.getDefinitionPath(definitionId, false); const relativePath = configUtils.getDefinitionPath(definitionId, true); - const dotDevContainerPath = path.join(definitionPath, '.devcontainer'); + const dotDevContainerPath = definitionPath; const devContainerJsonPath = path.join(dotDevContainerPath, 'devcontainer.json'); const devContainerJsonRaw = await asyncUtils.readFile(devContainerJsonPath); const devContainerJsonModified = @@ -198,71 +193,6 @@ async function updateScriptSources(devContainerDockerfileRaw, repo, release, upd return devContainerDockerfileModified; } -// Update script files and URLs in a Dockerfile to be release specific (or not) and optionally update the SHA to lock to this version -async function updateScriptSourcesInDockerfile(devContainerDockerfilePath, repo, release, updateScriptSha) { - const devContainerDockerfileRaw = await asyncUtils.readFile(devContainerDockerfilePath); - const devContainerDockerfileModified = await updateScriptSources(devContainerDockerfileRaw, repo, release, updateScriptSha); - await asyncUtils.writeFile(devContainerDockerfilePath, devContainerDockerfileModified); - await copyLibraryScriptsForDefinition(path.dirname(devContainerDockerfilePath)); -} - -// Update all script URLS in the entire repo (not staging folder) -async function updateAllScriptSourcesInRepo(repo, release, updateScriptSha) { - const definitionFolder = path.join(__dirname, '..', '..', 'containers'); - // Update script versions in definition Dockerfiles for release - const allDefinitions = await asyncUtils.readdir(definitionFolder, { withFileTypes: true }); - await asyncUtils.forEach(allDefinitions, async (currentDefinition) => { - if (!currentDefinition.isDirectory()) { - return; - } - - const dockerFileBasePath = path.join(definitionFolder, currentDefinition.name, '.devcontainer', 'base.Dockerfile'); - if (await asyncUtils.exists(dockerFileBasePath)) { - console.log(`(*) Looking for script source in base.Dockerfile for ${currentDefinition.name}.`); - await updateScriptSourcesInDockerfile(dockerFileBasePath, repo, release, updateScriptSha); - } - const dockerFilePath = path.join(definitionFolder, currentDefinition.name, '.devcontainer', 'Dockerfile'); - if (await asyncUtils.exists(dockerFilePath)) { - console.log(`(*) Looking for script source in Dockerfile for ${currentDefinition.name}.`); - await updateScriptSourcesInDockerfile(dockerFilePath, repo, release, updateScriptSha); - } - }); -} - -// Copy contents of script library to folder, meta.env file if specified and building -async function copyLibraryScriptsForDefinition(definitionDevContainerJsonFolder, isForBuild, meta) { - const libraryScriptsFolder = path.join(definitionDevContainerJsonFolder, scriptLibraryFolderNameInDefinition); - if (await asyncUtils.exists(libraryScriptsFolder)) { - await asyncUtils.forEach(await asyncUtils.readdir(libraryScriptsFolder), async (script) => { - // Only copy files that end in .sh - if (path.extname(script) !== '.sh') { - return; - } - const possibleScriptSource = path.join(__dirname, '..', '..', scriptLibraryPathInRepo, script); - if (await asyncUtils.exists(possibleScriptSource)) { - const targetScriptPath = path.join(libraryScriptsFolder, script); - console.log(`(*) Copying ${script} to ${libraryScriptsFolder}...`); - await asyncUtils.copyFile(possibleScriptSource, targetScriptPath); - } - }); - } - if (isForBuild && meta) { - // Write meta.env for use by scripts - metaEnvTemplate = metaEnvTemplate || handlebars.compile(await asyncUtils.readFile(path.join(__dirname, '..', 'assets', 'meta.env'))); - mkdirp(libraryScriptsFolder); - await asyncUtils.writeFile(path.join(libraryScriptsFolder, 'meta.env'), metaEnvTemplate(meta)); - } -} - -// For CI of the script library folder -async function copyLibraryScriptsForAllDefinitions() { - const devcontainerFolders = glob.sync(`${path.resolve(__dirname, '..', '..')}/+(containers|container-templates|repository-containers)/**/.devcontainer`); - await asyncUtils.forEach(devcontainerFolders, async (folder) => { - console.log(`(*) Checking ${path.basename(path.resolve(folder, '..'))} for ${scriptLibraryFolderNameInDefinition} folder...`); - await copyLibraryScriptsForDefinition(folder); - }); -} - function replaceFrom(dockerFileContents, newFromSection) { return dockerFileContents.replace(/(#\s+\[Choice\].+\n)?(ARG\s+VARIANT\s*=\s*.+\n)?(FROM\s+[^\s\n]+)/, newFromSection); } @@ -272,7 +202,4 @@ module.exports = { updateStub: updateStub, updateConfigForRelease: updateConfigForRelease, prepDockerFile: prepDockerFile, - copyLibraryScriptsForAllDefinitions: copyLibraryScriptsForAllDefinitions, - updateScriptSourcesInDockerfile: updateScriptSourcesInDockerfile, - updateAllScriptSourcesInRepo: updateAllScriptSourcesInRepo } diff --git a/build/src/push.js b/build/src/push.js index 38f621ed0..d5ff59885 100644 --- a/build/src/push.js +++ b/build/src/push.js @@ -37,10 +37,10 @@ async function push(repo, release, updateLatest, registry, registryPath, stubReg // See https://docs.docker.com/engine/reference/commandline/buildx_create/ console.log('(*) Setting up builder...'); const builders = await asyncUtils.exec('docker buildx ls'); - if(builders.indexOf('vscode-dev-containers') < 0) { - await asyncUtils.spawn('docker', ['buildx', 'create', '--use', '--name', 'vscode-dev-containers']); + if(builders.indexOf('devcontainers') < 0) { + await asyncUtils.spawn('docker', ['buildx', 'create', '--use', '--name', 'devcontainers']); } else { - await asyncUtils.spawn('docker', ['buildx', 'use', 'vscode-dev-containers']); + await asyncUtils.spawn('docker', ['buildx', 'use', 'devcontainers']); } // This step sets up the QEMU emulators for cross-platform builds. See https://github.com/docker/buildx#building-multi-platform-images await asyncUtils.spawn('docker', ['run', '--privileged', '--rm', 'tonistiigi/binfmt', '--install', 'all']); @@ -59,11 +59,9 @@ async function push(repo, release, updateLatest, registry, registryPath, stubReg async function pushImage(definitionId, repo, release, updateLatest, registry, registryPath, stubRegistry, stubRegistryPath, prepOnly, pushImages, replaceImage) { const definitionPath = configUtils.getDefinitionPath(definitionId); - const dotDevContainerPath = path.join(definitionPath, '.devcontainer'); - // Use base.Dockerfile for image build if found, otherwise use Dockerfile - const dockerFileExists = await asyncUtils.exists(path.join(dotDevContainerPath, 'Dockerfile')); - const baseDockerFileExists = await asyncUtils.exists(path.join(dotDevContainerPath, 'base.Dockerfile')); - const dockerFilePath = path.join(dotDevContainerPath, `${baseDockerFileExists ? 'base.' : ''}Dockerfile`); + const dotDevContainerPath = definitionPath; + // Use Dockerfile for image build + const dockerFilePath = path.join(dotDevContainerPath, 'Dockerfile'); // Make sure there's a Dockerfile present if (!await asyncUtils.exists(dockerFilePath)) { @@ -152,7 +150,7 @@ async function pushImage(definitionId, repo, release, updateLatest, '--label', `${imageLabelPrefix}.release=${prepResult.meta.gitRepositoryRelease}`, '--label', `${imageLabelPrefix}.source=${prepResult.meta.gitRepository}`, '--label', `${imageLabelPrefix}.timestamp='${prepResult.meta.buildTimestamp}'`, - '--builder', 'vscode-dev-containers', + '--builder', 'devcontainers', '--progress', 'plain', '--platform', pushImages ? architectures.reduce((prev, current) => prev + ',' + current, '').substring(1) : localArchitecture, pushImages ? '--push' : '--load', @@ -167,18 +165,8 @@ async function pushImage(definitionId, repo, release, updateLatest, } } - // If base.Dockerfile found, update stub/devcontainer.json, otherwise create - just use the default (first) variant if one exists - if (baseDockerFileExists && dockerFileExists) { - await prep.updateStub( - dotDevContainerPath, definitionId, repo, release, baseDockerFileExists, stubRegistry, stubRegistryPath); - console.log('(*) Updating devcontainer.json...'); - await asyncUtils.writeFile(devContainerJsonPath, devContainerJsonRaw.replace('"base.Dockerfile"', '"Dockerfile"')); - console.log('(*) Removing base.Dockerfile...'); - await asyncUtils.rimraf(dockerFilePath); - } else { - await prep.createStub( - dotDevContainerPath, definitionId, repo, release, baseDockerFileExists, stubRegistry, stubRegistryPath); - } + await prep.createStub( + dotDevContainerPath, definitionId, repo, release, false, stubRegistry, stubRegistryPath); console.log('(*) Done!\n'); } @@ -193,14 +181,14 @@ async function flattenBaseImage(baseImageTag, flattenedBaseImageTag, pushImages) // Flatten const processOpts = { stdio: 'inherit', shell: true }; console.log('(*) Preparing base image...'); - await asyncUtils.spawn('docker', ['run', '-d', '--name', 'vscode-dev-containers-build-flatten', baseImageTag, 'bash'], processOpts); - const containerInspectOutput = await asyncUtils.spawn('docker', ['inspect', 'vscode-dev-containers-build-flatten'], { shell: true, stdio: 'pipe' }); + await asyncUtils.spawn('docker', ['run', '-d', '--name', 'devcontainers-build-flatten', baseImageTag, 'bash'], processOpts); + const containerInspectOutput = await asyncUtils.spawn('docker', ['inspect', 'devcontainers-build-flatten'], { shell: true, stdio: 'pipe' }); console.log('(*) Flattening (this could take a while)...'); const config = JSON.parse(containerInspectOutput)[0].Config; const envString = config.Env.reduce((prev, current) => prev + ' ' + current, ''); const importArgs = `-c 'ENV ${envString}' -c 'ENTRYPOINT ${JSON.stringify(config.Entrypoint)}' -c 'CMD ${JSON.stringify(config.Cmd)}'`; - await asyncUtils.exec(`docker export vscode-dev-containers-build-flatten | docker import ${importArgs} - ${flattenedBaseImageTag}`, processOpts); - await asyncUtils.spawn('docker', ['container', 'rm', '-f', 'vscode-dev-containers-build-flatten'], processOpts); + await asyncUtils.exec(`docker export devcontainers-build-flatten | docker import ${importArgs} - ${flattenedBaseImageTag}`, processOpts); + await asyncUtils.spawn('docker', ['container', 'rm', '-f', 'devcontainers-build-flatten'], processOpts); // Push if enabled if (pushImages) { diff --git a/build/src/utils/config.js b/build/src/utils/config.js index 58fa76952..d51339a90 100644 --- a/build/src/utils/config.js +++ b/build/src/utils/config.js @@ -25,7 +25,7 @@ async function loadConfig(repoPath) { const definitionBuildConfigFile = getConfig('definitionBuildConfigFile', 'definition-manifest.json'); // Get list of definition folders - const containersPath = path.join(repoPath, getConfig('containersPathInRepo', 'containers')); + const containersPath = path.join(repoPath, 'src'); const definitions = await asyncUtils.readdir(containersPath, { withFileTypes: true }); await asyncUtils.forEach(definitions, async (definitionFolder) => { // If directory entry is a file (like README.md, skip @@ -54,19 +54,6 @@ async function loadConfig(repoPath) { } }); - // Load repo containers to build - const repoContainersToBuildPath = path.join(repoPath, getConfig('repoContainersToBuildPath', 'repository-containers/build')); - const repoContainerManifestFiles = glob.sync(`${repoContainersToBuildPath}/**/${definitionBuildConfigFile}`); - await asyncUtils.forEach(repoContainerManifestFiles, async (manifestFilePath) => { - const definitionPath = path.resolve(path.dirname(manifestFilePath)); - const definitionId = path.relative(repoContainersToBuildPath, definitionPath); - allDefinitionPaths[definitionId] = { - path: definitionPath, - relativeToRootPath: path.relative(repoPath, definitionPath) - } - await loadDefinitionManifest(manifestFilePath, definitionId); - }); - // Populate image variants and tag lookup for (let definitionId in config.definitionBuildSettings) { const buildSettings = config.definitionBuildSettings[definitionId]; @@ -588,7 +575,7 @@ function getFallbackPoolUrl(package) { async function getStagingFolder(release) { if (!stagingFolders[release]) { - const stagingFolder = path.join(os.tmpdir(), 'vscode-dev-containers', release); + const stagingFolder = path.join(os.tmpdir(), 'devcontainers', release); console.log(`(*) Copying files to ${stagingFolder}\n`); await asyncUtils.rimraf(stagingFolder); // Clean out folder if it exists await asyncUtils.mkdirp(stagingFolder); // Create the folder diff --git a/build/src/utils/image-content-extractor.js b/build/src/utils/image-content-extractor.js index 1acf4d9e8..336966bf2 100644 --- a/build/src/utils/image-content-extractor.js +++ b/build/src/utils/image-content-extractor.js @@ -501,7 +501,7 @@ async function getGoPackageInfo(imageTagOrContainerName, packages) { console.log(`(*) Gathering information about go modules and packages...`); const componentList = []; - const packageInstallOutput = await getCommandOutputFromContainer(imageTagOrContainerName, "cat /usr/local/etc/vscode-dev-containers/go.log"); + const packageInstallOutput = await getCommandOutputFromContainer(imageTagOrContainerName, "cat /usr/local/etc/devcontainers/go.log"); for(let package in packages) { if (typeof package === 'string') { const versionCommand = packages[package]; diff --git a/build/vscdc b/build/vscdc index 90cc94866..f2860bc8f 100644 --- a/build/vscdc +++ b/build/vscdc @@ -13,14 +13,14 @@ const generateImageInformationFiles = require('./src/image-info').generateImageI const configUtils = require('./src/utils/config'); const packageJson = require('../package.json'); -console.log('vscode-dev-containers CLI\nCopyright (c) Microsoft Corporation. All rights reserved.\n') +console.log('devcontainers CLI\nCopyright (c) Microsoft Corporation. All rights reserved.\n') require('yargs') .command('pack', 'package dev container definitions', (yargs) => { yargs .options({ 'release': { - describe: 'vscode-dev-containers release tag or a branch', + describe: 'devcontainerss release tag or a branch', default: `v${packageJson.version}` }, 'registry': { @@ -29,7 +29,7 @@ require('yargs') }, 'registry-path': { describe: 'container registry path', - default: configUtils.getConfig('containerRegistryPath', 'vscode/devcontainers') + default: configUtils.getConfig('containerRegistryPath', 'devcontainers') }, 'stub-registry': { describe: 'registry to add to stub', @@ -42,8 +42,8 @@ require('yargs') configUtils.getConfig('containerRegistryPath', '')) }, 'github-repo': { - describe: 'vscode-dev-containers repo name', - default: configUtils.getConfig('githubRepoName', 'microsoft/vscode-dev-containers') + describe: 'devcontainerss repo name', + default: configUtils.getConfig('githubRepoName', 'microsoft/devcontainerss') }, 'package-only': { describe: 'whether to prep/build/push before packaging', @@ -80,7 +80,7 @@ require('yargs') }) .options({ 'release': { - describe: 'vscode-dev-containers release tag or branch', + describe: 'devcontainerss release tag or branch', default: `v${packageJson.version}` }, 'registry': { @@ -102,8 +102,8 @@ require('yargs') configUtils.getConfig('containerRegistryPath', '')) }, 'github-repo': { - describe: 'vscode-dev-containers repo name', - default: configUtils.getConfig('githubRepoName', 'microsoft/vscode-dev-containers') + describe: 'devcontainerss repo name', + default: configUtils.getConfig('githubRepoName', 'microsoft/devcontainerss') }, 'update-latest': { describe: 'whether to tag latest and {MAJOR} if pushing', @@ -142,23 +142,6 @@ require('yargs') } }) }, pushCommand) - .command('update-script-sources ', 'updates all script source URLs in Dockerfiles to a tag or branch', (yargs) => { - yargs - .positional('release', { - describe: 'release tag to branch use for script URLs', - }) - .options({ - 'github-repo': { - describe: 'vscode-dev-containers repo name', - default: configUtils.getConfig('githubRepoName', 'microsoft/vscode-dev-containers') - }, - 'update-sha': { - describe: 'update script SHAs to match file', - type: 'boolean', - default: true - } - }) - }, updateScriptSourcesCommand) .command('cg [devcontainer]', 'generate cgmanifest.json', (yargs) => { yargs .positional('devcontainer', { @@ -167,7 +150,7 @@ require('yargs') }) .options({ 'release': { - describe: 'vscode-dev-containers release tag or branch', + describe: 'devcontainerss release tag or branch', default: 'main' }, 'registry': { @@ -189,8 +172,8 @@ require('yargs') configUtils.getConfig('containerRegistryPath', '')) }, 'github-repo': { - describe: 'vscode-dev-containers repo name', - default: configUtils.getConfig('githubRepoName', 'microsoft/vscode-dev-containers') + describe: 'devcontainerss repo name', + default: configUtils.getConfig('githubRepoName', 'microsoft/devcontainerss') }, 'build': { describe: 'whether to to build the image first step', @@ -231,7 +214,7 @@ require('yargs') }) .options({ 'release': { - describe: 'vscode-dev-containers release tag or branch', + describe: 'devcontainerss release tag or branch', default: 'main' }, 'registry': { @@ -253,8 +236,8 @@ require('yargs') configUtils.getConfig('containerRegistryPath', '')) }, 'github-repo': { - describe: 'vscode-dev-containers repo name', - default: configUtils.getConfig('githubRepoName', 'microsoft/vscode-dev-containers') + describe: 'devcontainerss repo name', + default: configUtils.getConfig('githubRepoName', 'microsoft/devcontainerss') }, 'build': { describe: 'whether to to build the image first step', @@ -309,7 +292,6 @@ require('yargs') } }) }, patchCommand) - .command('copy-library-scripts', 'copy files from script-library folder into appropriate definitions', () => {}, copyLibraryScriptsCommand) .demandCommand() .help() .argv; @@ -338,28 +320,6 @@ function packCommand(argv) { }); } -function updateScriptSourcesCommand(argv) { - prep.updateAllScriptSourcesInRepo(argv.githubRepo, argv.release, argv.updateSha) - .catch((reason) => { - console.error(`(!) Failed to update script sources - ${reason}`); - if(reason.stack) { - console.error(` ${reason.stack}`); - } - process.exit(1); - }); -} - -function copyLibraryScriptsCommand() { - prep.copyLibraryScriptsForAllDefinitions() - .catch((reason) => { - console.error(`(!) Failed to copy library scripts to definitions - ${reason}`); - if(reason.stack) { - console.error(` ${reason.stack}`); - } - process.exit(1); - }); -} - function imageInfoCommand(argv) { generateImageInformationFiles(argv.githubRepo, argv.release, argv.registry, argv.registryPath, argv.stubRegistry, argv.stubRegistryPath, argv.build, argv.prune, argv.cg, argv.markdown, argv.overwrite, argv.outputPath, argv.devcontainer)