Implement initial tool for publishing to crates.io (#251)

* Implement initial tool for publishing to crates.io

* CR feedback

* Fix clippy lints

* Add tools to CI

* Only run SDK CI when changing the SDK

* Revert "Only run SDK CI when changing the SDK"

.github/workflows/ci.yaml

@@ -1,7 +1,7 @@
 on: [ pull_request ]
 
 env:
-  rust_version: 1.52.1
+  rust_version: 1.53.0
 
 name: CI
 

.github/workflows/tool-ci.yaml

@@ -0,0 +1,39 @@
+on:
+  pull_request:
+    paths: 'tools/**'
+
+env:
+  rust_version: 1.53.0
+  rust_toolchain_components: clippy,rustfmt
+
+name: Tools CI
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    name: Compile, Test, and Lint the `tools/` path
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        name: Cargo Cache
+        with:
+          path: |
+            ~/.cargo/registry
+            ~/.cargo/git
+            tools/publisher/target
+          key: tools-${{ runner.os }}-cargo-${{ hashFiles('tools/**/Cargo.toml') }}
+          restore-keys: |
+            tools-${{ runner.os }}-cargo-
+      - uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.rust_version }}
+          components: ${{ env.rust_toolchain_components }}
+          default: true
+      - name: Format Check
+        run: rustfmt --check --edition 2018 $(find tools -name '*.rs' -print | grep -v /target/)
+      - name: Cargo Test
+        run: cargo test
+        working-directory: tools/publisher
+      - name: Cargo Clippy
+        run: cargo clippy -- -D warnings
+        working-directory: tools/publisher

tools/publisher/Cargo.toml

@@ -0,0 +1,21 @@
+[package]
+name = "publisher"
+version = "0.1.0"
+authors = ["AWS Rust SDK Team <[email protected]>"]
+description = "Tool used to publish the AWS SDK to crates.io"
+edition = "2018"
+license = "Apache-2.0"
+publish = false
+
+[dependencies]
+anyhow = "1.0"
+cargo_toml = "0.10.1"
+clap = "2.33"
+dialoguer = "0.8"
+num_cpus = "1.13"
+semver = "1.0"
+thiserror = "1.0"
+tokio = { version = "1.12", features = ["full"] }
+toml = "0.5.8"
+tracing = "0.1.29"
+tracing-subscriber = "0.2.25"

tools/publisher/README.md

@@ -0,0 +1 @@
+This is a tool that the SDK developer team uses to publish the AWS SDK to crates.io.

tools/publisher/src/cargo.rs

@@ -0,0 +1,86 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+//! Module for interacting with Cargo.
+
+use anyhow::{Context, Result};
+use std::path::{Path, PathBuf};
+use std::process::{Command, Output};
+
+macro_rules! cmd {
+    [ $( $x:expr ),* ] => {
+        {
+            let mut cmd = Cmd::new();
+            $(cmd.push($x);)*
+            cmd
+        }
+    };
+}
+
+/// Confirms that cargo exists on the path.
+pub async fn confirm_installed_on_path() -> Result<()> {
+    cmd!["cargo", "--version"]
+        .spawn()
+        .await
+        .context("cargo is not installed on the PATH")?;
+    Ok(())
+}
+
+/// Returns a `Cmd` that, when spawned, will asynchronously run `cargo publish` in the given crate path.
+pub fn publish_task(crate_path: &Path) -> Cmd {
+    cmd!["cargo", "publish"].working_dir(crate_path)
+}
+
+#[derive(Default)]
+pub struct Cmd {
+    parts: Vec<String>,
+    working_dir: Option<PathBuf>,
+}
+
+impl Cmd {
+    fn new() -> Cmd {
+        Default::default()
+    }
+
+    fn push(&mut self, part: impl Into<String>) {
+        self.parts.push(part.into());
+    }
+
+    fn working_dir(mut self, working_dir: impl AsRef<Path>) -> Self {
+        self.working_dir = Some(working_dir.as_ref().into());
+        self
+    }
+
+    /// Returns a plan string that can be output to the user to describe the command.
+    pub fn plan(&self) -> String {
+        let mut plan = String::new();
+        if let Some(working_dir) = &self.working_dir {
+            plan.push_str(&format!("[in {:?}]: ", working_dir));
+        }
+        plan.push_str(&self.parts.join(" "));
+        plan
+    }
+
+    /// Runs the command asynchronously.
+    pub async fn spawn(mut self) -> Result<Output> {
+        let working_dir = self
+            .working_dir
+            .take()
+            .unwrap_or_else(|| std::env::current_dir().unwrap());
+        let mut command: Command = self.into();
+        tokio::task::spawn_blocking(move || Ok(command.current_dir(working_dir).output()?)).await?
+    }
+}
+
+impl From<Cmd> for Command {
+    fn from(cmd: Cmd) -> Self {
+        assert!(!cmd.parts.is_empty());
+        let mut command = Command::new(&cmd.parts[0]);
+        for i in 1..cmd.parts.len() {
+            command.arg(&cmd.parts[i]);
+        }
+        command
+    }
+}

tools/publisher/src/fs.rs

@@ -0,0 +1,52 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+use anyhow::{Context, Result};
+use std::path::Path;
+use tokio::fs::File;
+use tokio::io::{AsyncReadExt, AsyncWriteExt};
+
+/// Abstraction of the filesystem to allow for more tests to be added in the future.
+#[derive(Clone, Debug)]
+pub enum Fs {
+    Real,
+}
+
+impl Fs {
+    /// Reads entire file into `Vec<u8>`
+    pub async fn read_file(&self, path: impl AsRef<Path>) -> Result<Vec<u8>> {
+        match self {
+            Fs::Real => tokio_read_file(path.as_ref()).await,
+        }
+    }
+
+    /// Writes an entire file from a `&[u8]`
+    pub async fn write_file(&self, path: impl AsRef<Path>, contents: &[u8]) -> Result<()> {
+        match self {
+            Fs::Real => tokio_write_file(path.as_ref(), contents).await,
+        }
+    }
+}
+
+async fn tokio_read_file(path: &Path) -> Result<Vec<u8>> {
+    let mut contents = Vec::new();
+    let mut file = File::open(path)
+        .await
+        .with_context(|| format!("failed to open {:?}", path))?;
+    file.read_to_end(&mut contents)
+        .await
+        .with_context(|| format!("failed to read {:?}", path))?;
+    Ok(contents)
+}
+
+async fn tokio_write_file(path: &Path, contents: &[u8]) -> Result<()> {
+    let mut file = File::create(path)
+        .await
+        .with_context(|| format!("failed to create {:?}", path))?;
+    file.write_all(contents)
+        .await
+        .with_context(|| format!("failed to write {:?}", path))?;
+    Ok(())
+}

tools/publisher/src/main.rs

@@ -0,0 +1,53 @@
+/*
+ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ * SPDX-License-Identifier: Apache-2.0.
+ */
+
+use crate::subcommand::fix_manifests::subcommand_fix_manifests;
+use crate::subcommand::publish::subcommand_publish;
+use anyhow::Result;
+use clap::{crate_authors, crate_description, crate_name, crate_version};
+
+mod cargo;
+mod fs;
+mod package;
+mod repo;
+mod sort;
+mod subcommand;
+
+pub const REPO_NAME: &str = "aws-sdk-rust";
+pub const REPO_CRATE_PATH: &str = "sdk";
+
+#[tokio::main]
+async fn main() -> Result<()> {
+    tracing_subscriber::fmt()
+        .with_env_filter(
+            std::env::var("RUST_LOG").unwrap_or_else(|_| "error,publisher=info".to_owned()),
+        )
+        .init();
+
+    let matches = clap_app().get_matches();
+    if let Some(_matches) = matches.subcommand_matches("publish") {
+        subcommand_publish().await?;
+    } else if let Some(_matches) = matches.subcommand_matches("fix-manifests") {
+        subcommand_fix_manifests().await?;
+    } else {
+        clap_app().print_long_help().unwrap();
+    }
+    Ok(())
+}
+
+fn clap_app() -> clap::App<'static, 'static> {
+    clap::App::new(crate_name!())
+        .version(crate_version!())
+        .author(crate_authors!())
+        .about(crate_description!())
+        // In the future, there may be another subcommand for yanking
+        .subcommand(
+            clap::SubCommand::with_name("fix-manifests")
+                .about("fixes path dependencies in manifests to also have version numbers"),
+        )
+        .subcommand(
+            clap::SubCommand::with_name("publish").about("publishes the AWS SDK to crates.io"),
+        )
+}