new: Support Depot for remote caching. (#1792)

* Add settings.

* More testing.

* Get things working.

* Add new action state.

* Update impls.

* Fix output.

* Fix tests.

* Fixes.

* Change types.

* Fix target state.

* Improve errors.

* Add backtrace.

* Try from_str.

* Fix tests.

* More debugging.

* Add docs.

* Polish.

* Bump versions.

.github/workflows/moon.yml

@@ -37,9 +37,11 @@ jobs:
           cache-base: '^(master|develop-)'
       - run: cargo run -- --color --log trace ci --base ${{ github.base_ref || 'master' }}
         env:
+          DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
           MOON_NODE_VERSION: ${{ matrix.node-version }}
           MOONBASE_SECRET_KEY: ${{ secrets.MOONBASE_SECRET_KEY }}
           MOONBASE_ACCESS_KEY: ${{ secrets.MOONBASE_ACCESS_KEY }}
+          RUST_BACKTRACE: '1'
       - uses: moonrepo/run-report-action@v1
         if: success() || failure()
         with:

.moon/workspace.yml

@@ -35,17 +35,22 @@ docker:
     include:
       - '*.config.js'
       - '*.json'
-# unstable_remote:
-#   host: 'http://0.0.0.0:8080'
-#   # host: 'grpc://0.0.0.0:9092'
-#   cache:
-#     compression: 'zstd'
-# mtls:
-#   caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem'
-#   clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem'
-#   clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key'
-#   domain: 'localhost'
-# tls:
-#   # assumeHttp2: true
-#   cert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem'
-#   # domain: 'localhost'
+
+unstable_remote:
+  host: 'grpcs://cache.depot.dev'
+  auth:
+    token: 'DEPOT_TOKEN'
+    headers:
+      'X-Depot-Org': '1xtpjd084j'
+      'X-Depot-Project': '90xxfkst9n'
+  # cache:
+  #   compression: 'zstd'
+  # mtls:
+  #   caCert: 'crates/remote/tests/__fixtures__/certs-local/ca.pem'
+  #   clientCert: 'crates/remote/tests/__fixtures__/certs-local/client.pem'
+  #   clientKey: 'crates/remote/tests/__fixtures__/certs-local/client.key'
+  #   domain: 'localhost'
+  # tls:
+  #   # assumeHttp2: true
+  #   cert: 'crates/remote/tests/__fixtures__/certs-local/client.pem'
+  #   domain: 'localhost'

.yarn/versions/c89cc34c.yml

@@ -1,9 +1,16 @@
 releases:
-  "@moonrepo/cli": minor
-  "@moonrepo/core-linux-arm64-gnu": minor
-  "@moonrepo/core-linux-arm64-musl": minor
-  "@moonrepo/core-linux-x64-gnu": minor
-  "@moonrepo/core-linux-x64-musl": minor
-  "@moonrepo/core-macos-arm64": minor
-  "@moonrepo/core-macos-x64": minor
-  "@moonrepo/core-windows-x64-msvc": minor
+  '@moonrepo/cli': minor
+  '@moonrepo/core-linux-arm64-gnu': minor
+  '@moonrepo/core-linux-arm64-musl': minor
+  '@moonrepo/core-linux-x64-gnu': minor
+  '@moonrepo/core-linux-x64-musl': minor
+  '@moonrepo/core-macos-arm64': minor
+  '@moonrepo/core-macos-x64': minor
+  '@moonrepo/core-windows-x64-msvc': minor
+  '@moonrepo/types': minor
+
+declined:
+  - '@moonrepo/nx-compat'
+  - '@moonrepo/report'
+  - '@moonrepo/runtime'
+  - website

CHANGELOG.md

@@ -1,5 +1,17 @@
 # Changelog
 
+## Unreleased
+
+#### 🚀 Updates
+
+- Updated our unstable remote service (Bazel RE API) with new functionality:
+  - You can now use `http(s)` protocols for gRPC servers, instead of just `grpc(s)`.
+  - Added an `unstable_remote.api` setting, which can be used to inform the server's API format.
+    Defaults to `grpc`.
+  - Added an `unstable_remote.auth` setting, which can be used for HTTP Bearer/token Authorization
+    based endpoints. Can also be used to set headers for all requests.
+  - Added support for Depot cloud-based caching: https://depot.dev/docs/cache/overview
+
 ## 1.31.3
 
 #### 🐞 Fixes

crates/config/src/workspace/remote_config.rs

@@ -1,4 +1,5 @@
 use crate::portable_path::FilePath;
+use rustc_hash::FxHashMap;
 use schematic::{derive_enum, validate, Config, ConfigEnum, ValidateError, ValidateResult};
 
 fn path_is_required<D, C>(
@@ -15,6 +16,27 @@ fn path_is_required<D, C>(
 }
 
 derive_enum!(
+    /// The API format of the remote service.
+    #[derive(Copy, ConfigEnum, Default)]
+    pub enum RemoteApi {
+        /// gRPC endpoints.
+        #[default]
+        Grpc,
+    }
+);
+
+/// Configures basic HTTP authentication.
+#[derive(Clone, Config, Debug)]
+pub struct RemoteAuthConfig {
+    /// HTTP headers to inject into every request.
+    pub headers: FxHashMap<String, String>,
+
+    /// The name of an environment variable to use as a bearer token.
+    pub token: Option<String>,
+}
+
+derive_enum!(
+    /// Supported blob compression levels.
     #[derive(Copy, ConfigEnum, Default)]
     pub enum RemoteCompression {
         /// No compression.
@@ -81,6 +103,13 @@ pub struct RemoteMtlsConfig {
 /// Configures the remote service, powered by the Bazel Remote Execution API.
 #[derive(Clone, Config, Debug)]
 pub struct RemoteConfig {
+    /// The API format of the remote service.
+    pub api: RemoteApi,
+
+    /// Connect to the host using basic HTTP authentication.
+    #[setting(nested)]
+    pub auth: Option<RemoteAuthConfig>,
+
     /// Configures the action cache (AC) and content addressable cache (CAS).
     #[setting(nested)]
     pub cache: RemoteCacheConfig,
@@ -101,11 +130,19 @@ pub struct RemoteConfig {
 }
 
 impl RemoteConfig {
+    pub fn is_bearer_auth(&self) -> bool {
+        self.auth.as_ref().is_some_and(|auth| auth.token.is_some())
+    }
+
     pub fn is_localhost(&self) -> bool {
         self.host.contains("localhost") || self.host.contains("0.0.0.0")
     }
 
     pub fn is_secure(&self) -> bool {
-        self.tls.is_some() || self.mtls.is_some()
+        self.is_bearer_auth() || self.tls.is_some() || self.mtls.is_some()
+    }
+
+    pub fn is_secure_protocol(&self) -> bool {
+        self.host.starts_with("https") || self.host.starts_with("grpcs")
     }
 }

crates/remote/Cargo.toml

@@ -8,8 +8,10 @@ publish = false
 moon_action = { path = "../action" }
 moon_common = { path = "../common" }
 moon_config = { path = "../config" }
+moon_task = { path = "../task" }
 async-trait = { workspace = true }
 bazel-remote-apis = { version = "0.12.0", features = ["serde"] }
+bincode = "1.3.3"
 chrono = { workspace = true }
 miette = { workspace = true }
 reqwest = { workspace = true, features = ["json"] }

crates/remote/src/action_state.rs

@@ -0,0 +1,145 @@
+use crate::fs_digest::*;
+use bazel_remote_apis::build::bazel::remote::execution::v2::{
+    command, platform, Action, ActionResult, Command, Digest, ExecutedActionMetadata,
+};
+use miette::IntoDiagnostic;
+use moon_action::Operation;
+use moon_task::Task;
+use std::collections::BTreeMap;
+use std::path::Path;
+
+pub struct ActionState<'task> {
+    task: &'task Task,
+
+    // RE API
+    pub action: Option<Action>,
+    pub action_result: Option<ActionResult>,
+    pub command: Option<Command>,
+    pub digest: Digest,
+
+    // To upload
+    pub blobs: Vec<Blob>,
+}
+
+impl ActionState<'_> {
+    pub fn new(digest: Digest, task: &Task) -> ActionState<'_> {
+        ActionState {
+            task,
+            action: None,
+            action_result: None,
+            command: None,
+            digest,
+            blobs: vec![],
+        }
+    }
+
+    pub fn create_action_from_task(&mut self) {
+        let mut action = Action {
+            command_digest: Some(self.digest.clone()),
+            do_not_cache: !self.task.options.cache,
+            input_root_digest: None, // TODO?
+            ..Default::default()
+        };
+
+        // https://github.com/bazelbuild/remote-apis/blob/main/build/bazel/remote/execution/v2/platform.md
+        if let Some(os_list) = &self.task.options.os {
+            let platform = action.platform.get_or_insert_default();
+
+            for os in os_list {
+                platform.properties.push(platform::Property {
+                    name: "OSFamily".into(),
+                    value: os.to_string(),
+                });
+            }
+        }
+
+        // Since we don't support (or plan to) remote execution,
+        // then we can ignore all the working directory logic
+        let mut command = Command {
+            arguments: vec![self.task.command.clone()],
+            output_paths: vec![], // TODO
+            ..Default::default()
+        };
+
+        command.arguments.extend(self.task.args.clone());
+
+        for (name, value) in BTreeMap::from_iter(self.task.env.clone()) {
+            command
+                .environment_variables
+                .push(command::EnvironmentVariable { name, value });
+        }
+
+        self.action = Some(action);
+        self.command = Some(command);
+    }
+
+    pub fn create_action_result_from_operation(
+        &mut self,
+        operation: &Operation,
+    ) -> miette::Result<()> {
+        let mut result = ActionResult {
+            execution_metadata: Some(ExecutedActionMetadata {
+                worker: "moon".into(),
+                execution_start_timestamp: create_timestamp_from_naive(operation.started_at),
+                execution_completed_timestamp: operation
+                    .finished_at
+                    .and_then(create_timestamp_from_naive),
+                ..Default::default()
+            }),
+            ..Default::default()
+        };
+
+        if let Some(exec) = operation.get_output() {
+            result.exit_code = exec.exit_code.unwrap_or_default();
+
+            if let Some(stderr) = &exec.stderr {
+                let blob = Blob::new(stderr.as_bytes().to_owned());
+
+                result.stderr_digest = Some(blob.digest.clone());
+                self.blobs.push(blob);
+            }
+
+            if let Some(stdout) = &exec.stdout {
+                let blob = Blob::new(stdout.as_bytes().to_owned());
+
+                result.stdout_digest = Some(blob.digest.clone());
+                self.blobs.push(blob);
+            }
+        }
+
+        self.action_result = Some(result);
+
+        Ok(())
+    }
+
+    pub fn set_action_result(&mut self, result: ActionResult) {
+        self.action_result = Some(result);
+    }
+
+    pub fn compute_outputs(&mut self, workspace_root: &Path) -> miette::Result<()> {
+        let mut outputs = OutputDigests::default();
+
+        for path in self.task.get_output_files(workspace_root, true)? {
+            outputs.insert_relative_path(path, workspace_root)?;
+        }
+
+        if let Some(result) = &mut self.action_result {
+            result.output_files = outputs.files;
+            result.output_symlinks = outputs.symlinks;
+            result.output_directories = outputs.dirs;
+            self.blobs.extend(outputs.blobs);
+        }
+
+        Ok(())
+    }
+
+    pub fn get_command_as_bytes(&self) -> miette::Result<Vec<u8>> {
+        bincode::serialize(&self.command).into_diagnostic()
+    }
+
+    pub fn prepare_for_upload(&mut self) -> Option<(ActionResult, Vec<Blob>)> {
+        self.action_result
+            .take()
+            .map(|result| (result, self.blobs.drain(0..).collect::<Vec<_>>()))
+    }
+}

crates/remote/src/compression.rs

@@ -23,6 +23,21 @@ pub fn get_compressor(compression: RemoteCompression) -> i32 {
     }
 }
 
+#[allow(dead_code)]
+pub fn get_compression_from_value(compressor: compressor::Value) -> RemoteCompression {
+    match compressor {
+        compressor::Value::Zstd => RemoteCompression::Zstd,
+        _ => RemoteCompression::None,
+    }
+}
+
+pub fn get_compression_from_code(compressor: i32) -> RemoteCompression {
+    match compressor {
+        zstd if zstd == compressor::Value::Zstd as i32 => RemoteCompression::Zstd,
+        _ => RemoteCompression::None,
+    }
+}
+
 pub fn compress_blob(compression: RemoteCompression, bytes: Vec<u8>) -> miette::Result<Vec<u8>> {
     let result = match compression {
         RemoteCompression::None => Ok(bytes),

crates/remote/src/fs_digest.rs

@@ -3,7 +3,7 @@
 use bazel_remote_apis::build::bazel::remote::execution::v2::{
     Digest, NodeProperties, OutputDirectory, OutputFile, OutputSymlink,
 };
-use bazel_remote_apis::google::protobuf::{Timestamp, UInt32Value};
+use bazel_remote_apis::google::protobuf::Timestamp;
 use chrono::NaiveDateTime;
 use moon_common::path::{PathExt, WorkspaceRelativePathBuf};
 use sha2::{Digest as Sha256Digest, Sha256};
@@ -14,7 +14,6 @@ use std::{
     path::{Path, PathBuf},
     time::{Duration, SystemTime, UNIX_EPOCH},
 };
-use tracing::instrument;
 
 pub struct Blob {
     pub bytes: Vec<u8>,
@@ -77,6 +76,7 @@ pub fn compute_node_properties(metadata: &Metadata) -> NodeProperties {
 
     #[cfg(unix)]
     {
+        use bazel_remote_apis::google::protobuf::UInt32Value;
         use std::os::unix::fs::PermissionsExt;
 
         props.unix_mode = Some(UInt32Value {
@@ -157,20 +157,6 @@ impl OutputDigests {
     }
 }
 
-#[instrument]
-pub fn compute_digests_for_outputs(
-    paths: Vec<WorkspaceRelativePathBuf>,
-    workspace_root: &Path,
-) -> miette::Result<OutputDigests> {
-    let mut result = OutputDigests::default();
-
-    for path in paths {
-        result.insert_relative_path(path, workspace_root)?;
-    }
-
-    Ok(result)
-}
-
 fn apply_node_properties(path: &Path, props: &NodeProperties) -> miette::Result<()> {
     if let Some(mtime) = &props.mtime {
         let modified = Duration::new(mtime.seconds as u64, mtime.nanos as u32);