HDDS-13668. Support S3 signed single chunk payload verification

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/ECKeyOutputStream.java

@@ -19,10 +19,12 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import jakarta.annotation.Nonnull;
 import java.io.IOException;
 import java.nio.Buffer;
 import java.nio.ByteBuffer;
 import java.util.Arrays;
+import java.util.Collections;
 import java.util.List;
 import java.util.Objects;
 import java.util.Set;
@@ -47,6 +49,7 @@
 import org.apache.ozone.erasurecode.rawcoder.RawErasureEncoder;
 import org.apache.ozone.erasurecode.rawcoder.util.CodecUtil;
 import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
+import org.apache.ratis.util.function.CheckedRunnable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -86,6 +89,13 @@ public final class ECKeyOutputStream extends KeyOutputStream
   // how much data has been ingested into the stream
   private long writeOffset;
 
+  private List<CheckedRunnable<IOException>> preCommits = Collections.emptyList();
+
+  @Override
+  public void setPreCommits(@Nonnull List<CheckedRunnable<IOException>> preCommits) {
+    this.preCommits = preCommits;
+  }
+
   @VisibleForTesting
   public void insertFlushCheckpoint(long version) throws IOException {
     addStripeToQueue(new CheckpointDummyStripe(version));
@@ -485,6 +495,9 @@ public void close() throws IOException {
               "Expected: %d and actual %d write sizes do not match",
                   expectedSize, offset));
         }
+        for (CheckedRunnable<IOException> preCommit : preCommits) {
+          preCommit.run();
+        }
         blockOutputStreamEntryPool.commitKey(offset);
       }
     } catch (ExecutionException e) {

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyDataStreamOutput.java

@@ -19,9 +19,11 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import jakarta.annotation.Nonnull;
 import java.io.IOException;
 import java.nio.ByteBuffer;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -44,6 +46,7 @@
 import org.apache.hadoop.ozone.om.helpers.OmMultipartCommitUploadPartInfo;
 import org.apache.hadoop.ozone.om.helpers.OpenKeySession;
 import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
+import org.apache.ratis.util.function.CheckedRunnable;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -82,6 +85,12 @@ public class KeyDataStreamOutput extends AbstractDataStreamOutput
    */
   private boolean atomicKeyCreation;
 
+  private List<CheckedRunnable<IOException>> preCommits = Collections.emptyList();
+
+  public void setPreCommits(@Nonnull List<CheckedRunnable<IOException>> preCommits) {
+    this.preCommits = preCommits;
+  }
+
   @VisibleForTesting
   public List<BlockDataStreamOutputEntry> getStreamEntries() {
     return blockDataStreamOutputEntryPool.getStreamEntries();
@@ -431,6 +440,9 @@ public void close() throws IOException {
             String.format("Expected: %d and actual %d write sizes do not match",
                 expectedSize, offset));
       }
+      for (CheckedRunnable<IOException> preCommit : preCommits) {
+        preCommit.run();
+      }
       blockDataStreamOutputEntryPool.commitKey(offset);
     } finally {
       blockDataStreamOutputEntryPool.cleanup();

hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStream.java

@@ -19,10 +19,12 @@
 
 import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
+import jakarta.annotation.Nonnull;
 import java.io.IOException;
 import java.io.InterruptedIOException;
 import java.io.OutputStream;
 import java.util.Collection;
+import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Objects;
@@ -110,6 +112,11 @@ public class KeyOutputStream extends OutputStream
 
   private final int maxConcurrentWritePerKey;
   private final KeyOutputStreamSemaphore keyOutputStreamSemaphore;
+  private List<CheckedRunnable<IOException>> preCommits = Collections.emptyList();
+
+  public void setPreCommits(@Nonnull List<CheckedRunnable<IOException>> preCommits) {
+    this.preCommits = preCommits;
+  }
 
   @VisibleForTesting
   KeyOutputStreamSemaphore getRequestSemaphore() {
@@ -655,6 +662,9 @@ private void closeInternal() throws IOException {
             String.format("Expected: %d and actual %d write sizes do not match",
                 expectedSize, offset));
       }
+      for (CheckedRunnable<IOException> preCommit : preCommits) {
+        preCommit.run();
+      }
       blockOutputStreamEntryPool.commitKey(offset);
     } finally {
       blockOutputStreamEntryPool.cleanup();

hadoop-ozone/dist/src/main/smoketest/s3/presigned_url_helper.py

@@ -0,0 +1,73 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import hashlib
+
+
+def generate_presigned_put_object_url(
+    aws_access_key_id=None,
+    aws_secret_access_key=None,
+    bucket_name=None,
+    object_key=None,
+    region_name='us-east-1',
+    expiration=3600,
+    content_type=None,
+    endpoint_url=None,
+):
+  """
+  Generate a presigned URL for PUT Object. This function creates the S3 client internally.
+  """
+  try:
+    import boto3
+
+    client_args = {
+      'service_name': 's3',
+      'region_name': region_name,
+    }
+
+    if aws_access_key_id and aws_secret_access_key:
+      client_args['aws_access_key_id'] = aws_access_key_id
+      client_args['aws_secret_access_key'] = aws_secret_access_key
+
+    if endpoint_url:
+      client_args['endpoint_url'] = endpoint_url
+
+    s3_client = boto3.client(**client_args)
+
+    params = {
+      'Bucket': bucket_name,
+      'Key': object_key,
+    }
+
+    if content_type:
+      params['ContentType'] = content_type
+
+    presigned_url = s3_client.generate_presigned_url(
+      ClientMethod='put_object',
+      Params=params,
+      ExpiresIn=expiration
+    )
+
+    return presigned_url
+
+  except Exception as e:
+    raise Exception(f"Failed to generate presigned URL: {str(e)}")
+
+
+def compute_sha256_file(path):
+  """Compute SHA256 hex digest for the entire file content at path."""
+  with open(path, 'rb') as f:
+    return hashlib.sha256(f.read()).hexdigest()

hadoop-ozone/dist/src/main/smoketest/s3/presignedurl.robot

@@ -0,0 +1,54 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+*** Settings ***
+Documentation       S3 gateway test with aws cli
+Library             OperatingSystem
+Library             String
+Library             ./presigned_url_helper.py
+Resource            ../commonlib.robot
+Resource            commonawslib.robot
+Test Timeout        5 minutes
+Suite Setup         Setup s3 tests
+
+*** Variables ***
+${ENDPOINT_URL}       http://s3g:9878
+${OZONE_TEST}         true
+${BUCKET}             generated
+
+*** Test Cases ***
+Presigned URL PUT Object
+    [Documentation]    Test presigned URL PUT object
+    Execute                  echo "Randomtext" > /tmp/testfile
+    ${ACCESS_KEY} =          Execute    aws configure get aws_access_key_id
+    ${SECRET_ACCESS_KEY} =   Execute    aws configure get aws_secret_access_key
+    ${presigned_url}=        Generate Presigned Put Object Url    ${ACCESS_KEY}    ${SECRET_ACCESS_KEY}    ${BUCKET}    test-presigned-put    us-east-1    3600    ${EMPTY}    ${ENDPOINT_URL}
+    ${SHA256} =              Compute Sha256 File    /tmp/testfile
+    ${result} =              Execute    curl -X PUT -T "/tmp/testfile" -H "x-amz-content-sha256: ${SHA256}" "${presigned_url}"
+    Should Not Contain       ${result}    Error
+    ${head_result} =         Execute AWSS3ApiCli    head-object --bucket ${BUCKET} --key test-presigned-put
+    Should Not Contain       ${head_result}    Error
+
+Presigned URL PUT Object using wrong x-amz-content-sha256
+    [Documentation]    Test presigned URL PUT object with wrong x-amz-content-sha256
+    Execute                  echo "Randomtext" > /tmp/testfile
+    ${ACCESS_KEY} =          Execute    aws configure get aws_access_key_id
+    ${SECRET_ACCESS_KEY} =   Execute    aws configure get aws_secret_access_key
+    ${presigned_url}=        Generate Presigned Put Object Url    ${ACCESS_KEY}    ${SECRET_ACCESS_KEY}    ${BUCKET}    test-presigned-put-wrong-sha    us-east-1    3600    ${EMPTY}    ${ENDPOINT_URL}
+    ${result} =              Execute    curl -X PUT -T "/tmp/testfile" -H "x-amz-content-sha256: wronghash" "${presigned_url}"
+    Should Contain           ${result}    The provided 'x-amz-content-sha256' header does not match the computed hash.
+    ${head_result} =         Execute AWSS3APICli and ignore error    head-object --bucket ${BUCKET} --key test-presigned-put-wrong-sha
+    Should contain           ${head_result}    404
+    Should contain           ${head_result}    Not Found

hadoop-ozone/integration-test-s3/pom.xml

@@ -66,6 +66,11 @@
       <artifactId>hadoop-common</artifactId>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>org.apache.kerby</groupId>
+      <artifactId>kerby-util</artifactId>
+      <scope>test</scope>
+    </dependency>
     <dependency>
       <groupId>org.apache.ozone</groupId>
       <artifactId>hdds-common</artifactId>

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java

@@ -1152,6 +1152,41 @@ public void testPresignedUrlPutObject() throws Exception {
       }
     }
 
+    @Test
+    public void testPresignedUrlPutSingleChunkWithWrongSha256() throws Exception {
+      final String keyName = getKeyName();
+
+      // Test PutObjectRequest presigned URL
+      GeneratePresignedUrlRequest generatePresignedUrlRequest =
+          new GeneratePresignedUrlRequest(BUCKET_NAME, keyName).withMethod(HttpMethod.PUT).withExpiration(expiration);
+      URL presignedUrl = s3Client.generatePresignedUrl(generatePresignedUrlRequest);
+
+      Map<String, List<String>> headers = new HashMap<>();
+      List<String> sha256Value = new ArrayList<>();
+      sha256Value.add("wrong-sha256-value");
+      headers.put("x-amz-content-sha256", sha256Value);
+
+      HttpURLConnection connection = null;
+      try {
+        connection = S3SDKTestUtils.openHttpURLConnection(presignedUrl, "PUT",
+            headers, CONTENT.getBytes(StandardCharsets.UTF_8));
+        int responseCode = connection.getResponseCode();
+        assertEquals(400, responseCode, "PutObject presigned URL should return 400 because of wrong SHA256");
+      } finally {
+        if (connection != null) {
+          connection.disconnect();
+        }
+      }
+
+      // Verify the object was not uploaded
+      AmazonServiceException ase = assertThrows(AmazonServiceException.class,
+          () -> s3Client.getObject(BUCKET_NAME, keyName));
+
+      assertEquals(ErrorType.Client, ase.getErrorType());
+      assertEquals(404, ase.getStatusCode());
+      assertEquals("NoSuchKey", ase.getErrorCode());
+    }
+
     @Test
     public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception {
       final String keyName = getKeyName();

hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v2/AbstractS3SDKV2Tests.java

@@ -67,6 +67,7 @@
 import org.apache.hadoop.ozone.s3.S3ClientFactory;
 import org.apache.hadoop.ozone.s3.awssdk.S3SDKTestUtils;
 import org.apache.hadoop.ozone.s3.endpoint.S3Owner;
+import org.apache.hadoop.ozone.s3.util.S3Consts;
 import org.apache.hadoop.security.UserGroupInformation;
 import org.apache.ozone.test.NonHATests;
 import org.apache.ozone.test.OzoneTestBase;
@@ -645,6 +646,41 @@ public void testPresignedUrlPut() throws Exception {
       assertEquals(CONTENT, actualContent);
     }
 
+    @Test
+    public void testPresignedUrlPutSingleChunkWithWrongSha256() throws Exception {
+      final String keyName = getKeyName();
+
+      PutObjectRequest objectRequest = PutObjectRequest.builder().bucket(BUCKET_NAME).key(keyName).build();
+
+      PutObjectPresignRequest presignRequest = PutObjectPresignRequest.builder()
+          .signatureDuration(duration)
+          .putObjectRequest(objectRequest)
+          .build();
+
+      PresignedPutObjectRequest presignedRequest = presigner.presignPutObject(presignRequest);
+
+      Map<String, List<String>> headers = presignedRequest.signedHeaders();
+      List<String> sha256 = new ArrayList<>();
+      sha256.add("wrong-sha256-value");
+      headers.put(S3Consts.X_AMZ_CONTENT_SHA256, sha256);
+
+      // use http url connection
+      HttpURLConnection connection = null;
+      try {
+        connection = S3SDKTestUtils.openHttpURLConnection(presignedRequest.url(), "PUT",
+            headers, CONTENT.getBytes(StandardCharsets.UTF_8));
+        int responseCode = connection.getResponseCode();
+        assertEquals(400, responseCode, "PutObject presigned URL should return 400 because of wrong SHA256");
+      } finally {
+        if (connection != null) {
+          connection.disconnect();
+        }
+      }
+
+      // Verify the object was not uploaded
+      assertThrows(NoSuchKeyException.class, () -> s3Client.headObject(b -> b.bucket(BUCKET_NAME).key(keyName)));
+    }
+
     @Test
     public void testPresignedUrlMultipartUpload(@TempDir Path tempDir) throws Exception {
       final String keyName = getKeyName();