Replace REST cache with Azure Cache2

.ci/build-platform.yml

@@ -11,7 +11,7 @@ jobs:
     steps: 
       - powershell: $Env:Path
         continueOnError: true
-        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        condition: eq(variables['AGENT.OS'], 'Windows_NT')
         displayName: "Print env in powershell"
       # Needed so that the mingw tar doesn't shadow the system tar. See
       # pipelines.yaml. We need windows bsdtar from system32, not the mingw
@@ -22,44 +22,16 @@ jobs:
         displayName: "Make sure windows/system32 is at front of path if windows"
       - powershell: $Env:Path
         continueOnError: true
-        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        condition: eq(variables['AGENT.OS'], 'Windows_NT')
         displayName: "Print env in powershell"
       - powershell: get-command tar
         continueOnError: true
-        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        condition: eq(variables['AGENT.OS'], 'Windows_NT')
         displayName: "Print where tar is located"
       - powershell: tar --help
         continueOnError: true
-        condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+        condition: eq(variables['AGENT.OS'], 'Windows_NT')
         displayName: "Print tar help"
-      - bash: |
-          # COMPUTE THE ESY INSTALL CACHE LOCATION AHEAD OF TIME
-          DESIRED_LEN="86"
-          # Note: This will need to change when upgrading esy version
-          # that reenables long paths on windows.
-          if [ "$AGENT_OS" == "Windows_NT" ]; then
-            DESIRED_LEN="33"
-          fi
-          HOME_ESY3="$HOME/.esy/3"
-          HOME_ESY3_LEN=${#HOME_ESY3}
-          NUM_UNDERS=$(echo "$(($DESIRED_LEN-$HOME_ESY3_LEN))")
-          UNDERS=$(printf "%-${NUM_UNDERS}s" "_")
-          UNDERS="${UNDERS// /_}"
-          THE_ESY__CACHE_INSTALL_PATH=${HOME_ESY3}${UNDERS}/i
-          if [ "$AGENT_OS" == "Windows_NT" ]; then
-            THE_ESY__CACHE_INSTALL_PATH=$( cygpath --mixed --absolute "$THE_ESY__CACHE_INSTALL_PATH")
-          fi
-          echo "THE_ESY__CACHE_INSTALL_PATH: $THE_ESY__CACHE_INSTALL_PATH"
-          # This will be exposed as an env var ESY__CACHE_INSTALL_PATH, or an
-          # Azure var esy__cache_install_path
-          echo "##vso[task.setvariable variable=esy__cache_install_path]$THE_ESY__CACHE_INSTALL_PATH"
-      # - bash: |
-      #     which esy
-      #     echo "$( which esy )"
-      #     echo "##vso[task.setvariable variable=esy_bin_location]$(which esy)"
-      #   displayName: "Find esy binary"
-      # - bash: echo ${ESY_BIN_LOCATION}
-      # displayName: "Print esy bin location"
       - bash: env
         displayName: "Print environment"
       - template: utils/use-node.yml

.ci/utils/publish-build-cache.yml

@@ -1,37 +1,7 @@
-# Steps for publishing project cache
-
 steps:
-  - bash: 'mkdir -p $(Build.StagingDirectory)'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
-    displayName: '[Cache][Publish] Create cache directory'
-
   # continueOnError because on windows it has a permission denied error but the
   # export succeeds.
   - script: "esy export-dependencies"
     continueOnError: true
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
+    condition: eq(variables.CACHE_RESTORED, 'false')
     displayName: "esy export-dependencies"
-
-  - bash: pwd && ls _export/* && mv _export '$(Build.StagingDirectory)' && ls '$(Build.StagingDirectory)/_export/'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
-    displayName: '[Cache][Publish] move export to staging dir'
-
-  # - bash: cd $ESY__CACHE_INSTALL_PATH && tar -czf $(Build.StagingDirectory)/esy-cache.tar .
-  #   workingDirectory: ''
-  #   condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
-  #   displayName: '[Cache][Publish] Tar esy cache directory'
-
-  # - bash: 'cd $(ESY__NPM_ROOT) && tar -czf $(Build.StagingDirectory)/npm-cache.tar .'
-  #   condition: and(succeeded(), eq(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))
-  #   displayName: '[Cache][Publish] Tar npm cache directory'
-
-  - task: PublishBuildArtifacts@1
-    displayName: '[Cache][Publish] Upload tarball'
-    condition: and(succeeded(), eq(variables['Build.Reason'], 'IndividualCI'))
-    # TODO: The CI Build caches are pulled down by the last successful buildID
-    # for the target branch.
-    inputs:
-        pathToPublish: '$(Build.StagingDirectory)'
-        artifactName: 'cache-$(Agent.OS)-install'
-        parallel: true
-        parallelCount: 8

.ci/utils/restore-build-cache.yml

@@ -1,101 +1,68 @@
-# Steps for restoring project cache
-
+# The cache key is built up of the following:
+# - We use a string that we can change to bust the cache
+# - The string "esy"
+# - The string for the OS
+# - The hash of the lock file
 steps:
-  - bash: 'mkdir -p $(Build.StagingDirectory)'
-    condition: and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch'])))
-    displayName: '[Cache][Publish] Create cache directory'
+  - bash: |
+      # COMPUTE THE ESY INSTALL CACHE LOCATION AHEAD OF TIME
+      # The cache location is only needed for some issues that require to remove
+      # individual packages from cache (e.g. ocamlfind below)
+      DESIRED_LEN="86"
+      # Note: This will need to change when upgrading esy version
+      # that reenables long paths on windows.
+      if [ "$AGENT_OS" == "Windows_NT" ]; then
+        DESIRED_LEN="33"
+      fi
+      HOME_ESY3="$HOME/.esy/3"
+      HOME_ESY3_LEN=${#HOME_ESY3}
+      NUM_UNDERS=$(echo "$(($DESIRED_LEN-$HOME_ESY3_LEN))")
+      UNDERS=$(printf "%-${NUM_UNDERS}s" "_")
+      UNDERS="${UNDERS// /_}"
+      THE_ESY__CACHE_INSTALL_PATH=${HOME_ESY3}${UNDERS}/i
+      if [ "$AGENT_OS" == "Windows_NT" ]; then
+        THE_ESY__CACHE_INSTALL_PATH=$( cygpath --mixed --absolute "$THE_ESY__CACHE_INSTALL_PATH")
+      fi
+      echo "THE_ESY__CACHE_INSTALL_PATH: $THE_ESY__CACHE_INSTALL_PATH"
+      # This will be exposed as an env var ESY__CACHE_INSTALL_PATH, or an
+      # Azure var esy__cache_install_path
+      echo "##vso[task.setvariable variable=esy__cache_install_path]$THE_ESY__CACHE_INSTALL_PATH"
+    displayName: '[Cache] calculate esy store path'
+
+  - task: Cache@2
+    inputs:
+      # "v1" prefix added just to keep the ability to clear a cache without having to wait 7 days
+      key: v1c | esy | $(Agent.OS) | esy.lock/index.json
+      restoreKeys: v1c | esy | $(Agent.OS)
+      path: _export
+      cacheHitVar: CACHE_RESTORED
+    displayName: '[Cache] esy packages'
 
-  # TODO: This can be done in parallel with installing node, and esy
-  # (which would save a bunch of time on windows)
+  # Remove as soon as https://github.com/esy/esy/pull/969 is resolved.
+  # For now, windows won't use build cache for problematic package.
   - task: Bash@3
-    condition: and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch'])))
-    displayName: '[Cache][Restore] Restoring build cache using REST API'
     continueOnError: true
+    condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), eq(variables.CACHE_RESTORED, 'true'))
+    displayName: 'Remove ocamlfind, rely and console prebuilts if on Windows'
     inputs:
       targetType: 'inline' # Optional. Options: filePath, inline
       script: |
-        # If org name is reasonml then REST_BASE will be: https://dev.azure.com/reasonml/
-        REST_BASE="${SYSTEM_TEAMFOUNDATIONCOLLECTIONURI}"
-        PROJ="$SYSTEM_TEAMPROJECT"
-        ART_NAME="cache-${AGENT_OS}-install"
-        fetchLatestBuild() {
-          PREFIX="branchName=refs%2Fheads%2F"
-          BRANCH=${PREFIX}${SYSTEM_PULLREQUEST_TARGETBRANCH}
-          FILTER='deletedFilter=excludeDeleted&statusFilter=completed&resultFilter=succeeded'
-          LATEST='queryOrder=finishTimeDescending&$top=1'
-          REST_BUILDS="$REST_BASE/$PROJ/_apis/build/builds?${FILTER}&${BRANCH}&${LATEST}&api-version=4.1"
-          echo "Rest call for builds: $REST_BUILDS"
-          REST_BUILDS_RESP=$(curl "$REST_BUILDS")
-          if [[ $REST_BUILDS_RESP =~ (\"web\":\{\"href\":\")([^\"]*) ]]; then LATEST_BUILD_PAGE="${BASH_REMATCH[2]}"; else LATEST_BUILD_PAGE=""; fi
-          if [[ $REST_BUILDS_RESP =~ (\"badge\":\{\"href\":\")([^\"]*) ]]; then LATEST_BUILD_BADGE="${BASH_REMATCH[2]}"; else LATEST_BUILD_BADGE=""; fi
-          if [[ $REST_BUILDS_RESP =~ (\"id\":)([^,]*) ]]; then LATEST_BUILD_ID="${BASH_REMATCH[2]}"; else LATEST_BUILD_ID=""; fi
-        }
-        fetchLatestBuild
-        fetchArtifactURL() {
-          REST_ART="$REST_BASE/$PROJ/_apis/build/builds/$LATEST_BUILD_ID/artifacts?artifactName=$ART_NAME&api-version=4.1"
-          echo "Rest call for artifacts: $REST_ART"
-          if [[ $(curl $REST_ART) =~ (downloadUrl\":\")([^\"]*) ]]; then LATEST_ART_URL="${BASH_REMATCH[2]}"; else LATEST_ART_URL=""; fi
-        }
-        downloadArtifactAndContinue() {
-          if [ -z "$LATEST_ART_URL" ]
-          then
-            echo "No latest artifact for merge-target branch found at URL $REST_ART"
-          else
-            curl "$LATEST_ART_URL" > "${BUILD_STAGINGDIRECTORY}/$ART_NAME.zip"
-            PROJECT_DIR=$PWD
-            cd $BUILD_STAGINGDIRECTORY
-            unzip "$ART_NAME.zip"
-            echo "Using Dependency cache for buildID: $LATEST_BUILD_ID"
-            echo "Build log for build that produced the cache: $LATEST_BUILD_PAGE"
-            echo "Build badge for build that produced the cache: $LATEST_BUILD_BADGE"
-            echo "Build artifact from build that produced the cache: $LATEST_ART_URL"
-            echo "Restoring build cache into:"
-            mkdir -p $ESY__CACHE_INSTALL_PATH
-            echo $ESY__CACHE_INSTALL_PATH
-            echo "##vso[task.setvariable variable=esy_export_dir_to_import]${BUILD_STAGINGDIRECTORY}/${ART_NAME}/_export"
-            if [[ -d ${ART_NAME}/_export ]]; then
-              echo "Cached builds to import from ${ART_NAME}/_export in next CI step"
-              ls ${ART_NAME}/_export
-              mv ${ART_NAME}/_export ${PROJECT_DIR}/_export
-            else
-              echo "No _export directory to import from build cache"
-              echo "Here's the contents of build cache:"
-              find ${BUILD_STAGINGDIRECTORY}
-            fi
-          fi
-        }
-        fetchArtifactURL
-        downloadArtifactAndContinue
-
+        ls _export
+        rm -r _export/opam__s__ocamlfind*
+        rm -r _export/reason_native__s__rely*
+        rm -r _export/reason_native__s__console*
+
   - powershell: esy.cmd import-dependencies
     continueOnError: true
-    condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
+    condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), eq(variables.CACHE_RESTORED, 'true'))
     displayName: "esy import-dependencies if windows (build cache from CI cache)"
 
   - bash: esy import-dependencies
     continueOnError: true
-    condition: and(ne(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
-    displayName: "esy import-dependencies if not windows (build cache from CI cache)"
-
-  # Remove as soon as https://github.com/esy/esy/pull/969 is resolved.
-  # For now, windows won't use build cache for problematic package.
-  - task: Bash@3
-    continueOnError: true
-    condition: and(eq(variables['AGENT.OS'], 'Windows_NT'), and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch']))))
-    displayName: 'Remove ocamlfind prebuilts if on Windows'
-    inputs:
-      targetType: 'inline' # Optional. Options: filePath, inline
-      script: |
-        ls ${ESY__CACHE_INSTALL_PATH}
-        rm -r ${ESY__CACHE_INSTALL_PATH}/opam__s__ocamlfind*
+    condition: and(ne(variables['AGENT.OS'], 'Windows_NT'), eq(variables.CACHE_RESTORED, 'true'))
+    displayName: "esy import-dependencies if not windows (build cache from CI cache)"    
 
   - bash: 'rm -rf _import'
     continueOnError: true
-    condition: and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch'])))
+    condition: eq(variables.CACHE_RESTORED, 'true')
     displayName: 'Remove import directory'
-
-  - bash: 'rm -rf *'
-    continueOnError: true
-    workingDirectory: '$(Build.StagingDirectory)'
-    condition: and(eq(variables['Build.Reason'], 'PullRequest'), and(succeeded(), ne(variables['Build.SourceBranch'], variables['System.PullRequest.TargetBranch'])))
-    displayName: '[Cache][Restore] Clean up staging dir'

.ci/utils/use-esy.yml

@@ -1,5 +1,5 @@
 # steps to install esy globally
 
 steps:
-  - script: "npm install -g [email protected].6"
+  - script: "npm install -g [email protected].8"
     displayName: "install esy"

README.md

@@ -68,9 +68,11 @@ no dependencies.
   - When asked how to configure the new pipeline, select the option to use
     existing configuration inside the repo.
 
-The CI is configured to build caches on the `master` branch, and also any
-branch named one of (`global`, `release-*`, `releases-*`). That means that pull
-requests to any branch with those names will be fast, once you have landed at
-least one commit to that branch. The first time you submit a pull request to
-one of those branches, the builds will be slow but then subsequent pull
-requests will be faster once a pull request is merged to it.
+The CI is configured to build caches on the `master` branch as well as any pull
+request run. Cache keys depend on the platform where builds are run (Linux,
+macOS or Windows) as well as the hash of the contents from `esy.lock` file.
+In case there is not a match for the current run key, there is a fallback to
+get the latest created key for the current run platform.
+Cache isolation and security applies thanks to Azure infrastructure. To know
+more about this, check the documentation about ["Cache isolation and
+security"](https://docs.microsoft.com/en-us/azure/devops/pipelines/caching/?view=azure-devops#cache-isolation-and-security).

azure-pipelines.yml

@@ -1,11 +1,14 @@
 name: Build npm release
 
 trigger:
-  - master
-  - global
-  - release-*
-  - releases-*
-  - feature-*
+  branches:
+    include:
+      - master
+
+pr:
+  branches:
+    include:
+      - "*"
 
 jobs:
   - template: .ci/build-platform.yml