Add HTTP User Detail extension

This adds functionality to unFTP that allows unFTP to obtain user
detail over HTTP in addition to the already existing JSON file.

It uses the extact same format as the JSON file functionality. The
'usr-http-url' command line arguments activate this feature. You
pass it a base URL, unFTP appends a username to it and performs
a GET request to the URL. The HTTP server should respond with a
200 OK and JSON body containing an array of users which should
at least contain the requested user's details.

Later on we can support diffent HTTP verbs and sending the
username via an HTTP header instead of the URL path or Post
body. For now I'm just keeping it simple.

Author: hannesdejager

.github/workflows/rust.yml

@@ -22,7 +22,7 @@ on:
     branches:
       - master
   release:
-    types: [created]
+    types: [ created ]
 
 jobs:
 
@@ -47,20 +47,20 @@ jobs:
     runs-on: ubuntu-latest
     if: ${{ github.ref != 'refs/heads/master' }}
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v3
-    - name: Install Rust toolchain
-      uses: actions-rs/toolchain@v1
-      with:
-        toolchain: ${{ env.RUST_VERSION }}
-        override: true
-        default: true
-        components: rustfmt
-    - name: Check formatting
-      uses: actions-rs/cargo@v1
-      with:
-        command: fmt
-        args: --all -- --check
+      - name: Checkout code
+        uses: actions/checkout@v3
+      - name: Install Rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+          override: true
+          default: true
+          components: rustfmt
+      - name: Check formatting
+        uses: actions-rs/cargo@v1
+        with:
+          command: fmt
+          args: --all -- --check
 
   clippy:
     name: Run Clippy
@@ -84,24 +84,24 @@ jobs:
           args: --all-features --workspace -- -D warnings
 
   test:
-      name: Run Tests
-      runs-on: ubuntu-latest
-      steps:
-        - name: Checkout sources
-          uses: actions/checkout@v3
-        - name: Install rust toolchain
-          uses: actions-rs/toolchain@v1
-          with:
-            toolchain: ${{ env.RUST_VERSION }}
-            override: true
-            default: true
-            components: clippy
-        - name: Install build dependencies
-          run: sudo apt-get update && sudo apt-get install -y libpam-dev
-        - name: Run tests
-          run: cargo test --verbose  --workspace --all --all-features
-        - name: Build Docs
-          run: cargo doc --all-features --workspace --no-deps
+    name: Run Tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3
+      - name: Install rust toolchain
+        uses: actions-rs/toolchain@v1
+        with:
+          toolchain: ${{ env.RUST_VERSION }}
+          override: true
+          default: true
+          components: clippy
+      - name: Install build dependencies
+        run: sudo apt-get update && sudo apt-get install -y libpam-dev
+      - name: Run tests
+        run: cargo test --verbose  --workspace --all --all-features
+      - name: Build Docs
+        run: cargo doc --all-features --workspace --no-deps
 
   build-linux-gnu:
     runs-on: ubuntu-latest
@@ -244,7 +244,7 @@ jobs:
           target: ${{ env.target }}
       - name: Install Rosetta
         if: runner.os == 'macOS' && runner.arch == 'arm64'
-        run: softwareupdate --install-rosetta          
+        run: softwareupdate --install-rosetta --agree-to-license
       - name: Build
         run: cargo build --release --target=${{ env.target }} --features rest_auth,jsonfile_auth,cloud_storage
       - name: Rename
@@ -255,7 +255,7 @@ jobs:
           name: unftp_${{ env.target }}
           path: target/${{ env.target }}/release/unftp_${{ env.target }}
 
-  upload-release-binaries:    
+  upload-release-binaries:
     if: ${{ github.event_name == 'release' }}  # Testing: if: ${{ github.ref == 'refs/heads/hannes/upload' }}
     runs-on: ubuntu-latest
     strategy:
@@ -329,7 +329,7 @@ jobs:
         run: chmod +x ./x86_64-unknown-linux-musl/unftp_x86_64-unknown-linux-musl
 
       - name: Build Docker image
-        run: docker build -t bolcom/unftp:${{ env.BUILD_VERSION }}-scratch -f packaging/docker/scratch.Dockerfile.ci . 
+        run: docker build -t bolcom/unftp:${{ env.BUILD_VERSION }}-scratch -f packaging/docker/scratch.Dockerfile.ci .
 
       - name: Save Docker image as tar
         run: docker save -o docker-image-scratch.tar bolcom/unftp:${{ env.BUILD_VERSION }}-scratch

Cargo.lock

@@ -52,6 +52,7 @@ unftp-auth-rest = { version = "0.2.4", optional = true }
 unftp-auth-jsonfile = { version = "0.3.3", optional = true }
 unftp-sbe-rooter = "0.2.0"
 unftp-sbe-restrict = "0.1.1"
+url = "2.5.0"
 
 [target.'cfg(unix)'.dependencies]
 unftp-auth-pam = { version = "0.2.4", optional = true }

Cargo.toml

@@ -42,6 +42,7 @@ pub const REDIS_PORT: &str = "log-redis-port";
 pub const ROOT_DIR: &str = "root-dir";
 pub const STORAGE_BACKEND_TYPE: &str = "sbe-type";
 pub const USR_JSON_PATH: &str = "usr-json-path";
+pub const USR_HTTP_URL: &str = "usr-http-url";
 pub const VERBOSITY: &str = "verbosity";
 
 #[derive(ArgEnum, Clone, Debug)]
@@ -502,6 +503,14 @@ pub(crate) fn clap_app(tmp_dir: &str) -> clap::Command {
                 .env("UNFTP_USR_JSON_PATH")
                 .takes_value(true),
         )
+        .arg(
+            Arg::new(USR_HTTP_URL)
+                .long("usr-http-url")
+                .value_name("URL")
+                .help("The URL to fetch user details from via a GET request. The username will be appended to this path.")
+                .env("UNFTP_USR_HTTP_URL")
+                .takes_value(true),
+        )
         .arg(
             Arg::new(PUBSUB_BASE_URL)
                 .long("ntf-pubsub-base-url")

src/args.rs

@@ -1,4 +1,4 @@
-use crate::domain::user::{User, UserDetailProvider};
+use crate::domain::user::{User, UserDetailError, UserDetailProvider};
 use async_trait::async_trait;
 use libunftp::auth::{AuthenticationError, Credentials, DefaultUser};
 
@@ -32,11 +32,10 @@ impl libunftp::auth::Authenticator<User> for LookupAuthenticator {
     ) -> Result<User, AuthenticationError> {
         self.inner.authenticate(username, creds).await?;
         let user_provider = self.usr_detail.as_ref().unwrap();
-        if let Some(user) = user_provider.provide_user_detail(username) {
-            Ok(user)
-        } else {
-            Ok(User::with_defaults(username))
-        }
+        Ok(user_provider
+            .provide_user_detail(username)
+            .await
+            .map_err(|e| AuthenticationError::with_source("error getting user detail", e))?)
     }
 
     async fn cert_auth_sufficient(&self, username: &str) -> bool {
@@ -47,8 +46,9 @@ impl libunftp::auth::Authenticator<User> for LookupAuthenticator {
 #[derive(Debug)]
 pub struct DefaultUserProvider {}
 
+#[async_trait]
 impl UserDetailProvider for DefaultUserProvider {
-    fn provide_user_detail(&self, username: &str) -> Option<User> {
-        Some(User::with_defaults(username))
+    async fn provide_user_detail(&self, username: &str) -> Result<User, UserDetailError> {
+        Ok(User::with_defaults(username))
     }
 }

src/auth.rs

@@ -1,6 +1,12 @@
+//! Contains definitions pertaining to FTP User Detail
+use async_trait::async_trait;
 use libunftp::auth::UserDetail;
-use std::fmt::{Debug, Display, Formatter};
-use std::path::PathBuf;
+use slog::error;
+use std::{
+    fmt::{Debug, Display, Formatter},
+    path::PathBuf,
+};
+use thiserror::Error;
 use unftp_sbe_restrict::{UserWithPermissions, VfsOperations};
 use unftp_sbe_rooter::UserWithRoot;
 
@@ -64,6 +70,39 @@ impl UserWithPermissions for User {
 
 /// Implementation of UserDetailProvider can look up and provide FTP user account details from
 /// a source.
+#[async_trait]
 pub trait UserDetailProvider: Debug {
-    fn provide_user_detail(&self, username: &str) -> Option<User>;
+    /// This will do the lookup. An error is returned if the user was not found or something else
+    /// went wrong.
+    async fn provide_user_detail(&self, username: &str) -> Result<User, UserDetailError>;
+}
+
+/// The error type returned by [`UserDetailProvider`]
+#[derive(Debug, Error)]
+pub enum UserDetailError {
+    #[error("{0}")]
+    Generic(String),
+    #[error("user '{username:?}' not found")]
+    UserNotFound { username: String },
+    #[error("error getting user details: {0}: {1:?}")]
+    ImplPropagated(
+        String,
+        #[source] Option<Box<dyn std::error::Error + Send + Sync + 'static>>,
+    ),
+}
+
+impl UserDetailError {
+    /// Creates a new domain specific error
+    #[allow(dead_code)]
+    pub fn new(s: impl Into<String>) -> Self {
+        UserDetailError::ImplPropagated(s.into(), None)
+    }
+
+    /// Creates a new domain specific error with the given source error.
+    pub fn with_source<E>(s: impl Into<String>, source: E) -> Self
+    where
+        E: std::error::Error + Send + Sync + 'static,
+    {
+        UserDetailError::ImplPropagated(s.into(), Some(Box::new(source)))
+    }
 }

src/domain/user.rs

@@ -1,8 +1,8 @@
 //! Infra contains infrastructure specific implementations of things in the [`domain`](crate::domain)
 //! module.
 mod pubsub;
-mod workload_identity;
-
+pub mod userdetail_http;
 pub mod usrdetail_json;
+mod workload_identity;
 
 pub use pubsub::PubsubEventDispatcher;

src/infra/mod.rs

@@ -0,0 +1,69 @@
+//! A libunftp [`UserDetail`](libunftp::auth::user::UserDetail) provider that obtains user detail
+//! over HTTP.
+
+use crate::domain::user::{User, UserDetailError, UserDetailProvider};
+use crate::infra::usrdetail_json::JsonUserProvider;
+use async_trait::async_trait;
+use http::{Method, Request};
+use hyper::{Body, Client};
+use url::form_urlencoded;
+
+/// A libunftp [`UserDetail`](libunftp::auth::user::UserDetail) provider that obtains user detail
+/// over HTTP.
+#[derive(Debug)]
+pub struct HTTPUserDetailProvider {
+    url: String,
+    #[allow(dead_code)]
+    header_name: Option<String>,
+}
+
+impl HTTPUserDetailProvider {
+    /// Creates a provider that will obtain user detail from the specified URL.
+    pub fn new(url: impl Into<String>) -> HTTPUserDetailProvider {
+        HTTPUserDetailProvider {
+            url: url.into(),
+            header_name: None,
+        }
+    }
+}
+
+impl Default for HTTPUserDetailProvider {
+    fn default() -> Self {
+        HTTPUserDetailProvider {
+            url: "http://localhost:8080/users/".to_string(),
+            header_name: None,
+        }
+    }
+}
+
+#[async_trait]
+impl UserDetailProvider for HTTPUserDetailProvider {
+    async fn provide_user_detail(&self, username: &str) -> Result<User, UserDetailError> {
+        let _url_suffix: String = form_urlencoded::byte_serialize(username.as_bytes()).collect();
+        let req = Request::builder()
+            .method(Method::GET)
+            .header("Content-type", "application/json")
+            .uri(format!("{}{}", self.url, username))
+            .body(Body::empty())
+            .map_err(|e| UserDetailError::with_source("error creating request", e))?;
+
+        let client = Client::new();
+
+        let resp = client
+            .request(req)
+            .await
+            .map_err(|e| UserDetailError::with_source("error doing HTTP request", e))?;
+
+        let body_bytes = hyper::body::to_bytes(resp.into_body())
+            .await
+            .map_err(|e| UserDetailError::with_source("error parsing body", e))?;
+
+        let json_str = std::str::from_utf8(body_bytes.as_ref())
+            .map_err(|e| UserDetailError::with_source("body is not a valid UTF string", e))?;
+
+        let json_usr_provider =
+            JsonUserProvider::from_json(json_str).map_err(UserDetailError::Generic)?;
+
+        json_usr_provider.provide_user_detail(username).await
+    }
+}