From 818037b8b361b2ae62f455617f49835826448759 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Sun, 21 Dec 2025 15:40:28 +0800 Subject: [PATCH 1/3] Support Range header in HeadObject --- .../ozone/s3/endpoint/ObjectEndpoint.java | 38 +++++- .../ozone/s3/endpoint/TestObjectHead.java | 108 +++++++++++++++++- 2 files changed, 137 insertions(+), 9 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index c6a2b6539098..eebfb7d16ba5 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -606,6 +606,7 @@ public Response head( S3GAction s3GAction = S3GAction.HEAD_KEY; OzoneKey key; + RangeHeader rangeHeader = null; try { if (S3Owner.hasBucketOwnershipVerificationConditions(getHeaders())) { OzoneBucket bucket = getBucket(bucketName); @@ -614,7 +615,6 @@ public Response head( key = getClientProtocol().headS3Object(bucketName, keyPath); isFile(keyPath, key); - // TODO: return the specified range bytes of this object. } catch (OMException ex) { auditReadFailure(s3GAction, ex); getMetrics().updateHeadKeyFailureStats(startNanos); @@ -637,10 +637,38 @@ public Response head( S3StorageType.STANDARD : S3StorageType.fromReplicationConfig(key.getReplicationConfig()); - ResponseBuilder response = Response.ok().status(HttpStatus.SC_OK) - .header(HttpHeaders.CONTENT_LENGTH, key.getDataSize()) - .header(HttpHeaders.CONTENT_TYPE, "binary/octet-stream") - .header(STORAGE_CLASS_HEADER, s3StorageType.toString()); + long length = key.getDataSize(); + String rangeHeaderVal = getHeaders() != null ? + getHeaders().getHeaderString(RANGE_HEADER) : null; + + // Parse Range header if present + if (rangeHeaderVal != null) { + rangeHeader = RangeHeaderParserUtil.parseRangeHeader(rangeHeaderVal, length); + if (rangeHeader.isInValidRange()) { + throw newError(S3ErrorTable.INVALID_RANGE, rangeHeaderVal); + } + } + + ResponseBuilder response; + + if (rangeHeaderVal == null || rangeHeader.isReadFull()) { + response = Response.ok().status(HttpStatus.SC_OK) + .header(HttpHeaders.CONTENT_LENGTH, length); + } else { + long startOffset = rangeHeader.getStartOffset(); + long endOffset = rangeHeader.getEndOffset(); + long contentLength = endOffset - startOffset + 1; + String contentRangeVal = RANGE_HEADER_SUPPORTED_UNIT + " " + + startOffset + "-" + endOffset + "/" + length; + + response = Response.status(Status.PARTIAL_CONTENT) + .header(HttpHeaders.CONTENT_LENGTH, contentLength) + .header(CONTENT_RANGE_HEADER, contentRangeVal); + } + + response.header(HttpHeaders.CONTENT_TYPE, "binary/octet-stream") + .header(STORAGE_CLASS_HEADER, s3StorageType.toString()) + .header(ACCEPT_RANGE_HEADER, RANGE_HEADER_SUPPORTED_UNIT); String eTag = key.getMetadata().get(OzoneConsts.ETAG); if (eTag != null) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index acecf7f81a87..48598945f0f5 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -20,12 +20,18 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.when; import java.io.IOException; import java.time.format.DateTimeFormatter; import java.util.HashMap; +import javax.ws.rs.core.HttpHeaders; import javax.ws.rs.core.Response; import org.apache.commons.lang3.RandomStringUtils; import org.apache.hadoop.hdds.client.ReplicationConfig; @@ -48,20 +54,21 @@ public class TestObjectHead { private String bucketName = "b1"; private ObjectEndpoint keyEndpoint; private OzoneBucket bucket; + private OzoneClient client; @BeforeEach public void setup() throws IOException { //Create client stub and object store stub. - OzoneClient clientStub = new OzoneClientStub(); + client = new OzoneClientStub(); // Create volume and bucket - clientStub.getObjectStore().createS3Bucket(bucketName); + client.getObjectStore().createS3Bucket(bucketName); - bucket = clientStub.getObjectStore().getS3Bucket(bucketName); + bucket = client.getObjectStore().getS3Bucket(bucketName); // Create HeadBucket and setClient to OzoneClientStub keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() - .setClient(clientStub) + .setClient(client) .build(); } @@ -185,4 +192,97 @@ public void testHeadWhenKeyIsAFileAndKeyPathEndsWithASlash() assertEquals(HttpStatus.SC_NOT_FOUND, response.getStatus()); bucket.deleteKey(keyPath); } + + @Test + public void testHeadWithRangeHeader() throws Exception { + //GIVEN + String value = "0123456789"; + OzoneOutputStream out = bucket.createKey("key1", + value.getBytes(UTF_8).length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.ONE), new HashMap<>()); + out.write(value.getBytes(UTF_8)); + out.close(); + + HttpHeaders headers = mock(HttpHeaders.class); + when(headers.getHeaderString(RANGE_HEADER)).thenReturn("bytes=0-0"); + keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() + .setClient(client) + .setHeaders(headers) + .build(); + + //WHEN + Response response = keyEndpoint.head(bucketName, "key1"); + + //THEN + assertEquals(206, response.getStatus()); + assertEquals("1", response.getHeaderString("Content-Length")); + assertEquals(String.format("bytes 0-0/%d", value.length()), + response.getHeaderString("Content-Range")); + assertEquals("bytes", response.getHeaderString("Accept-Ranges")); + + // Test range from start to end + when(headers.getHeaderString(RANGE_HEADER)).thenReturn("bytes=0-"); + response = keyEndpoint.head(bucketName, "key1"); + assertEquals(206, response.getStatus()); + assertEquals(String.valueOf(value.length()), + response.getHeaderString("Content-Length")); + assertEquals(String.format("bytes 0-%d/%d", value.length() - 1, value.length()), + response.getHeaderString("Content-Range")); + + bucket.deleteKey("key1"); + } + + @Test + public void testHeadWithInvalidRangeHeader() throws Exception { + //GIVEN + String value = "0123456789"; // length = 10 + OzoneOutputStream out = bucket.createKey("key1", + value.getBytes(UTF_8).length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.ONE), new HashMap<>()); + out.write(value.getBytes(UTF_8)); + out.close(); + + HttpHeaders headers = mock(HttpHeaders.class); + // Invalid range: both start and end are beyond file length + // According to RangeHeaderParserUtil, bytes=11-10 with length=10 will trigger isInValidRange() + when(headers.getHeaderString(RANGE_HEADER)).thenReturn("bytes=11-10"); + keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() + .setClient(client) + .setHeaders(headers) + .build(); + + //WHEN/THEN + OS3Exception ex = assertThrows(OS3Exception.class, + () -> keyEndpoint.head(bucketName, "key1")); + assertEquals("InvalidRange", ex.getCode()); + assertEquals(416, ex.getHttpCode()); + + bucket.deleteKey("key1"); + } + + @Test + public void testHeadWithoutRangeHeader() throws Exception { + //GIVEN + String value = "0123456789"; + OzoneOutputStream out = bucket.createKey("key1", + value.getBytes(UTF_8).length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.ONE), new HashMap<>()); + out.write(value.getBytes(UTF_8)); + out.close(); + + //WHEN + Response response = keyEndpoint.head(bucketName, "key1"); + + //THEN + assertEquals(200, response.getStatus()); + assertEquals(String.valueOf(value.length()), + response.getHeaderString("Content-Length")); + assertEquals("bytes", response.getHeaderString("Accept-Ranges")); + assertNull(response.getHeaderString("Content-Range")); + + bucket.deleteKey("key1"); + } } From 90deff2e1c4787dc3f3110b29441cf2b6c745e00 Mon Sep 17 00:00:00 2001 From: rich7420 Date: Mon, 29 Dec 2025 17:26:12 +0800 Subject: [PATCH 2/3] update --- .../ozone/s3/endpoint/ObjectEndpoint.java | 3 +- .../ozone/s3/endpoint/TestObjectHead.java | 119 ++++++++---------- 2 files changed, 50 insertions(+), 72 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java index eebfb7d16ba5..eb23b1223a3c 100644 --- a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java +++ b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/ObjectEndpoint.java @@ -606,7 +606,6 @@ public Response head( S3GAction s3GAction = S3GAction.HEAD_KEY; OzoneKey key; - RangeHeader rangeHeader = null; try { if (S3Owner.hasBucketOwnershipVerificationConditions(getHeaders())) { OzoneBucket bucket = getBucket(bucketName); @@ -641,7 +640,7 @@ public Response head( String rangeHeaderVal = getHeaders() != null ? getHeaders().getHeaderString(RANGE_HEADER) : null; - // Parse Range header if present + RangeHeader rangeHeader = null; if (rangeHeaderVal != null) { rangeHeader = RangeHeaderParserUtil.parseRangeHeader(rangeHeaderVal, length); if (rangeHeader.isInValidRange()) { diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index 48598945f0f5..d725f78dbdcc 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -20,6 +20,8 @@ import static java.net.HttpURLConnection.HTTP_NOT_FOUND; import static java.nio.charset.StandardCharsets.UTF_8; import static org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED; +import static org.apache.hadoop.ozone.s3.util.S3Consts.ACCEPT_RANGE_HEADER; +import static org.apache.hadoop.ozone.s3.util.S3Consts.CONTENT_RANGE_HEADER; import static org.apache.hadoop.ozone.s3.util.S3Consts.RANGE_HEADER; import static org.assertj.core.api.Assertions.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -39,11 +41,12 @@ import org.apache.hadoop.hdds.client.ReplicationType; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.client.OzoneBucket; -import org.apache.hadoop.ozone.client.OzoneClient; import org.apache.hadoop.ozone.client.OzoneClientStub; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.s3.exception.OS3Exception; +import org.apache.hadoop.ozone.s3.exception.S3ErrorTable; import org.apache.http.HttpStatus; +import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -51,27 +54,46 @@ * Test head object. */ public class TestObjectHead { + private static final String TEST_KEY = "key1"; + private static final String TEST_VALUE = "0123456789"; + private String bucketName = "b1"; private ObjectEndpoint keyEndpoint; private OzoneBucket bucket; - private OzoneClient client; + private HttpHeaders headers; @BeforeEach public void setup() throws IOException { - //Create client stub and object store stub. - client = new OzoneClientStub(); - - // Create volume and bucket + OzoneClientStub client = new OzoneClientStub(); client.getObjectStore().createS3Bucket(bucketName); - bucket = client.getObjectStore().getS3Bucket(bucketName); - // Create HeadBucket and setClient to OzoneClientStub + try (OzoneOutputStream out = bucket.createKey(TEST_KEY, + TEST_VALUE.getBytes(UTF_8).length, + ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, + ReplicationFactor.ONE), new HashMap<>())) { + out.write(TEST_VALUE.getBytes(UTF_8)); + } + + headers = mock(HttpHeaders.class); + when(headers.getHeaderString(RANGE_HEADER)).thenReturn(null); keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() .setClient(client) + .setHeaders(headers) .build(); } + @AfterEach + public void cleanup() throws IOException { + if (bucket != null) { + try { + bucket.deleteKey(TEST_KEY); + } catch (Exception e) { + // Ignore errors during cleanup to ensure test isolation + } + } + } + @Test public void testHeadObject() throws Exception { //GIVEN @@ -196,93 +218,50 @@ public void testHeadWhenKeyIsAFileAndKeyPathEndsWithASlash() @Test public void testHeadWithRangeHeader() throws Exception { //GIVEN - String value = "0123456789"; - OzoneOutputStream out = bucket.createKey("key1", - value.getBytes(UTF_8).length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, - ReplicationFactor.ONE), new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - - HttpHeaders headers = mock(HttpHeaders.class); when(headers.getHeaderString(RANGE_HEADER)).thenReturn("bytes=0-0"); - keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() - .setClient(client) - .setHeaders(headers) - .build(); //WHEN - Response response = keyEndpoint.head(bucketName, "key1"); + Response response = keyEndpoint.head(bucketName, TEST_KEY); //THEN assertEquals(206, response.getStatus()); - assertEquals("1", response.getHeaderString("Content-Length")); - assertEquals(String.format("bytes 0-0/%d", value.length()), - response.getHeaderString("Content-Range")); - assertEquals("bytes", response.getHeaderString("Accept-Ranges")); + assertEquals("1", response.getHeaderString(HttpHeaders.CONTENT_LENGTH)); + assertEquals(String.format("bytes 0-0/%d", TEST_VALUE.length()), + response.getHeaderString(CONTENT_RANGE_HEADER)); + assertEquals("bytes", response.getHeaderString(ACCEPT_RANGE_HEADER)); - // Test range from start to end + // Test range from start to end of file when(headers.getHeaderString(RANGE_HEADER)).thenReturn("bytes=0-"); - response = keyEndpoint.head(bucketName, "key1"); + response = keyEndpoint.head(bucketName, TEST_KEY); assertEquals(206, response.getStatus()); - assertEquals(String.valueOf(value.length()), - response.getHeaderString("Content-Length")); - assertEquals(String.format("bytes 0-%d/%d", value.length() - 1, value.length()), - response.getHeaderString("Content-Range")); - - bucket.deleteKey("key1"); + assertEquals(String.valueOf(TEST_VALUE.length()), + response.getHeaderString(HttpHeaders.CONTENT_LENGTH)); + assertEquals(String.format("bytes 0-%d/%d", TEST_VALUE.length() - 1, TEST_VALUE.length()), + response.getHeaderString(CONTENT_RANGE_HEADER)); } @Test public void testHeadWithInvalidRangeHeader() throws Exception { - //GIVEN - String value = "0123456789"; // length = 10 - OzoneOutputStream out = bucket.createKey("key1", - value.getBytes(UTF_8).length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, - ReplicationFactor.ONE), new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - - HttpHeaders headers = mock(HttpHeaders.class); - // Invalid range: both start and end are beyond file length - // According to RangeHeaderParserUtil, bytes=11-10 with length=10 will trigger isInValidRange() + // Invalid range: start (11) and end (10) exceed file length (10) when(headers.getHeaderString(RANGE_HEADER)).thenReturn("bytes=11-10"); - keyEndpoint = EndpointBuilder.newObjectEndpointBuilder() - .setClient(client) - .setHeaders(headers) - .build(); //WHEN/THEN OS3Exception ex = assertThrows(OS3Exception.class, - () -> keyEndpoint.head(bucketName, "key1")); - assertEquals("InvalidRange", ex.getCode()); + () -> keyEndpoint.head(bucketName, TEST_KEY)); + assertEquals(S3ErrorTable.INVALID_RANGE.getCode(), ex.getCode()); assertEquals(416, ex.getHttpCode()); - - bucket.deleteKey("key1"); } @Test public void testHeadWithoutRangeHeader() throws Exception { - //GIVEN - String value = "0123456789"; - OzoneOutputStream out = bucket.createKey("key1", - value.getBytes(UTF_8).length, - ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, - ReplicationFactor.ONE), new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); - //WHEN - Response response = keyEndpoint.head(bucketName, "key1"); + Response response = keyEndpoint.head(bucketName, TEST_KEY); //THEN assertEquals(200, response.getStatus()); - assertEquals(String.valueOf(value.length()), - response.getHeaderString("Content-Length")); - assertEquals("bytes", response.getHeaderString("Accept-Ranges")); - assertNull(response.getHeaderString("Content-Range")); - - bucket.deleteKey("key1"); + assertEquals(String.valueOf(TEST_VALUE.length()), + response.getHeaderString(HttpHeaders.CONTENT_LENGTH)); + assertEquals("bytes", response.getHeaderString(ACCEPT_RANGE_HEADER)); + assertNull(response.getHeaderString(CONTENT_RANGE_HEADER)); } } From 742154249966bb5ca55aed16b8719bcf9ee6426b Mon Sep 17 00:00:00 2001 From: rich7420 Date: Mon, 29 Dec 2025 17:29:28 +0800 Subject: [PATCH 3/3] update --- .../hadoop/ozone/s3/endpoint/TestObjectHead.java | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java index d725f78dbdcc..ac077a4c10d7 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/endpoint/TestObjectHead.java @@ -98,24 +98,26 @@ public void cleanup() throws IOException { public void testHeadObject() throws Exception { //GIVEN String value = RandomStringUtils.secure().nextAlphanumeric(32); - OzoneOutputStream out = bucket.createKey("key1", + String testKey = "testKey"; + try (OzoneOutputStream out = bucket.createKey(testKey, value.getBytes(UTF_8).length, ReplicationConfig.fromTypeAndFactor(ReplicationType.RATIS, - ReplicationFactor.ONE), new HashMap<>()); - out.write(value.getBytes(UTF_8)); - out.close(); + ReplicationFactor.ONE), new HashMap<>())) { + out.write(value.getBytes(UTF_8)); + } //WHEN - Response response = keyEndpoint.head(bucketName, "key1"); + Response response = keyEndpoint.head(bucketName, testKey); //THEN assertEquals(200, response.getStatus()); assertEquals(value.getBytes(UTF_8).length, - Long.parseLong(response.getHeaderString("Content-Length"))); + Long.parseLong(response.getHeaderString(HttpHeaders.CONTENT_LENGTH))); DateTimeFormatter.RFC_1123_DATE_TIME .parse(response.getHeaderString("Last-Modified")); + bucket.deleteKey(testKey); } @Test