Support unauthenticated access to public buckets

Use of a basic authentication header in requests to the Reductionist API
is now optional. If basic auth credentials are provided, they will be
used as S3 access/secret keys as before. Otherwise, requests to S3 will
be unauthenticated. Attempts to access a private bucket without
authentication will return a 401.

Author: markgoddard

benches/s3_client.rs

@@ -6,7 +6,7 @@ use aws_types::region::Region;
 use axum::body::Bytes;
 use criterion::{black_box, criterion_group, criterion_main, Criterion};
 use reductionist::resource_manager::ResourceManager;
-use reductionist::s3_client::{S3Client, S3ClientMap};
+use reductionist::s3_client::{S3Client, S3ClientMap, S3Credentials};
 use url::Url;
 // Bring trait into scope to use as_bytes method.
 use zerocopy::AsBytes;
@@ -40,6 +40,7 @@ fn criterion_benchmark(c: &mut Criterion) {
     let url = Url::parse("http://localhost:9000").unwrap();
     let username = "minioadmin";
     let password = "minioadmin";
+    let credentials = S3Credentials::access_key(username, password);
     let bucket = "s3-client-bench";
     let runtime = tokio::runtime::Runtime::new().unwrap();
     let map = S3ClientMap::new();
@@ -53,7 +54,7 @@ fn criterion_benchmark(c: &mut Criterion) {
         let name = format!("s3_client({})", size);
         c.bench_function(&name, |b| {
             b.to_async(&runtime).iter(|| async {
-                let client = S3Client::new(&url, username, password).await;
+                let client = S3Client::new(&url, credentials.clone()).await;
                 client
                     .download_object(black_box(bucket), &key, None, &resource_manager, &mut None)
                     .await
@@ -63,7 +64,7 @@ fn criterion_benchmark(c: &mut Criterion) {
         let name = format!("s3_client_map({})", size);
         c.bench_function(&name, |b| {
             b.to_async(&runtime).iter(|| async {
-                let client = map.get(&url, username, password).await;
+                let client = map.get(&url, credentials.clone()).await;
                 client
                     .download_object(black_box(bucket), &key, None, &resource_manager, &mut None)
                     .await

docs/api.md

@@ -73,6 +73,7 @@ The request body should be a JSON object of the form:
 ```
 
 Request authentication is implemented using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) with the username and password consisting of your S3 Access Key ID and Secret Access Key, respectively.
+Unauthenticated access to S3 is possible by omitting the basic auth header.
 
 On success, all operations return HTTP 200 OK with the response using the same datatype as specified in the request except for `count` which always returns the result as `int64`.
 The server returns the following headers with the HTTP response:

docs/contributing.md

@@ -150,7 +150,7 @@ Proxy functionality can be tested using the [S3 active storage compliance suite]
 
 ### Making requests to active storage endpoints
 
-Request authentication is implemented using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) with the username and password consisting of your S3 Access Key ID and Secret Access Key, respectively. These credentials are then used internally to authenticate with the upstream S3 source using [standard AWS authentication methods](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html)
+Request authentication is implemented using [Basic Auth](https://en.wikipedia.org/wiki/Basic_access_authentication) with the username and password consisting of your S3 Access Key ID and Secret Access Key, respectively. If provided, these credentials are then used internally to authenticate with the upstream S3 source using [standard AWS authentication methods](https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-auth-using-authorization-header.html). If no basic auth header is provided, an unauthenticated request will be made to S3.
 
 A basic Python client is provided in `scripts/client.py`.
 First install dependencies in a Python virtual environment:

src/app.rs

@@ -14,12 +14,10 @@ use crate::validated_json::ValidatedJson;
 
 use axum::middleware;
 use axum::{
-    body::{Body, Bytes},
+    body::Bytes,
     extract::{Path, State},
     headers::authorization::{Authorization, Basic},
     http::header,
-    http::Request,
-    http::StatusCode,
     response::{IntoResponse, Response},
     routing::{get, post},
     Router, TypedHeader,
@@ -31,7 +29,6 @@ use tower::Layer;
 use tower::ServiceBuilder;
 use tower_http::normalize_path::NormalizePathLayer;
 use tower_http::trace::TraceLayer;
-use tower_http::validate_request::ValidateRequestHeaderLayer;
 use tracing::debug_span;
 use tracing::Instrument;
 
@@ -113,8 +110,6 @@ pub fn init(args: &CommandLineArgs) {
 /// The router is populated with all routes as well as the following middleware:
 ///
 /// * a [tower_http::trace::TraceLayer] for tracing requests and responses
-/// * a [tower_http::validate_request::ValidateRequestHeaderLayer] for validating authorisation
-///   headers
 fn router(args: &CommandLineArgs) -> Router {
     fn v1(state: SharedAppState) -> Router {
         Router::new()
@@ -124,20 +119,7 @@ fn router(args: &CommandLineArgs) -> Router {
             .route("/select", post(operation_handler::<operations::Select>))
             .route("/sum", post(operation_handler::<operations::Sum>))
             .route("/:operation", post(unknown_operation_handler))
-            .layer(
-                ServiceBuilder::new()
-                    .layer(TraceLayer::new_for_http())
-                    .layer(ValidateRequestHeaderLayer::custom(
-                        // Validate that an authorization header has been provided.
-                        |request: &mut Request<Body>| {
-                            if request.headers().contains_key(header::AUTHORIZATION) {
-                                Ok(())
-                            } else {
-                                Err(StatusCode::UNAUTHORIZED.into_response())
-                            }
-                        },
-                    )),
-            )
+            .layer(ServiceBuilder::new().layer(TraceLayer::new_for_http()))
             .with_state(state)
     }
 
@@ -183,7 +165,7 @@ async fn schema() -> &'static str {
 ///
 /// # Arguments
 ///
-/// * `auth`: Basic authentication credentials
+/// * `client`: S3 client object
 /// * `request_data`: RequestData object for the request
 #[tracing::instrument(
     level = "DEBUG",
@@ -220,18 +202,23 @@ async fn download_object<'a>(
 ///
 /// # Arguments
 ///
-/// * `auth`: Basic authorization header
+/// * `auth`: Optional basic authentication header
 /// * `request_data`: RequestData object for the request
 async fn operation_handler<T: operation::Operation>(
     State(state): State<SharedAppState>,
-    TypedHeader(auth): TypedHeader<Authorization<Basic>>,
+    auth: Option<TypedHeader<Authorization<Basic>>>,
     ValidatedJson(request_data): ValidatedJson<models::RequestData>,
 ) -> Result<models::Response, ActiveStorageError> {
     let memory = request_data.size.unwrap_or(0);
     let mut _mem_permits = state.resource_manager.memory(memory).await?;
+    let credentials = if let Some(TypedHeader(auth)) = auth {
+        s3_client::S3Credentials::access_key(auth.username(), auth.password())
+    } else {
+        s3_client::S3Credentials::None
+    };
     let s3_client = state
         .s3_client_map
-        .get(&request_data.source, auth.username(), auth.password())
+        .get(&request_data.source, credentials)
         .instrument(tracing::Span::current())
         .await;
     let data = download_object(

src/s3_client.rs

@@ -13,17 +13,36 @@ use tokio::sync::{RwLock, SemaphorePermit};
 use tracing::Instrument;
 use url::Url;
 
+#[derive(Clone, Eq, Hash, PartialEq)]
+pub enum S3Credentials {
+    AccessKey {
+        access_key: String,
+        secret_key: String,
+    },
+    None,
+}
+
+impl S3Credentials {
+    /// Create an access key credential.
+    pub fn access_key(access_key: &str, secret_key: &str) -> Self {
+        S3Credentials::AccessKey {
+            access_key: access_key.to_string(),
+            secret_key: secret_key.to_string(),
+        }
+    }
+}
+
 /// A map containing initialised S3Client objects.
 ///
 /// The [aws_sdk_s3::Client] object is relatively expensive to create, so we reuse them where
 /// possible. This type provides a map for storing the clients objects.
 ///
-/// The map's key is a 3-tuple of the S3 URL, username and password.
+/// The map's key is a 2-tuple of the S3 URL and credentials.
 /// The value is the corresponding client object.
 pub struct S3ClientMap {
     /// A [hashbrown::HashMap] for storing the S3 clients. A read-write lock synchronises access to
     /// the map, optimised for reads.
-    map: RwLock<HashMap<(Url, String, String), S3Client>>,
+    map: RwLock<HashMap<(Url, S3Credentials), S3Client>>,
 }
 
 // FIXME: Currently clients are never removed from the map. If a large number of endpoints or
@@ -43,10 +62,9 @@ impl S3ClientMap {
     /// # Arguments
     ///
     /// * `url`: Object storage API URL
-    /// * `username`: Object storage account username
-    /// * `password`: Object storage account password
-    pub async fn get(&self, url: &Url, username: &str, password: &str) -> S3Client {
-        let key = (url.clone(), username.to_string(), password.to_string());
+    /// * `credentials`: Object storage account credentials
+    pub async fn get(&self, url: &Url, credentials: S3Credentials) -> S3Client {
+        let key = (url.clone(), credentials.clone());
         // Common case: return an existing client from the map.
         {
             let map = self.map.read().await;
@@ -61,7 +79,7 @@ impl S3ClientMap {
             client.clone()
         } else {
             tracing::info!("Creating new S3 client for {}", url);
-            let client = S3Client::new(url, username, password).await;
+            let client = S3Client::new(url, credentials).await;
             let (_, client) = map.insert_unique_unchecked(key, client);
             client.clone()
         }
@@ -81,13 +99,21 @@ impl S3Client {
     /// # Arguments
     ///
     /// * `url`: Object storage API URL
-    /// * `username`: Object storage account username
-    /// * `password`: Object storage account password
-    pub async fn new(url: &Url, username: &str, password: &str) -> Self {
-        let credentials = Credentials::from_keys(username, password, None);
+    /// * `credentials`: Object storage account credentials
+    pub async fn new(url: &Url, credentials: S3Credentials) -> Self {
         let region = Region::new("us-east-1");
-        let s3_config = aws_sdk_s3::Config::builder()
-            .credentials_provider(credentials)
+        let builder = aws_sdk_s3::Config::builder();
+        let builder = match credentials {
+            S3Credentials::AccessKey {
+                access_key,
+                secret_key,
+            } => {
+                let credentials = Credentials::from_keys(access_key, secret_key, None);
+                builder.credentials_provider(credentials)
+            }
+            S3Credentials::None => builder,
+        };
+        let s3_config = builder
             .region(Some(region))
             .endpoint_url(url.to_string())
             .force_path_style(true)
@@ -178,21 +204,38 @@ mod tests {
     use super::*;
     use url::Url;
 
+    fn make_access_key() -> S3Credentials {
+        S3Credentials::access_key("user", "password")
+    }
+
+    fn make_alt_access_key() -> S3Credentials {
+        S3Credentials::access_key("user2", "password")
+    }
+
     #[tokio::test]
     async fn s3_client_map() {
         let url = Url::parse("http://example.com").unwrap();
         let map = S3ClientMap::new();
-        map.get(&url, "user", "password").await;
-        map.get(&url, "user", "password").await;
+        map.get(&url, make_access_key()).await;
+        map.get(&url, make_access_key()).await;
         assert_eq!(map.map.read().await.len(), 1);
-        map.get(&url, "user2", "password2").await;
+        map.get(&url, make_alt_access_key()).await;
         assert_eq!(map.map.read().await.len(), 2);
+        map.get(&url, S3Credentials::None).await;
+        map.get(&url, S3Credentials::None).await;
+        assert_eq!(map.map.read().await.len(), 3);
     }
 
     #[tokio::test]
     async fn new() {
         let url = Url::parse("http://example.com").unwrap();
-        S3Client::new(&url, "user", "password").await;
+        S3Client::new(&url, make_access_key()).await;
+    }
+
+    #[tokio::test]
+    async fn new_no_auth() {
+        let url = Url::parse("http://example.com").unwrap();
+        S3Client::new(&url, S3Credentials::None).await;
     }
 
     #[test]