Skip to content

Adds gzip handler to JDK http client based on header #35225

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 2 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,10 +24,15 @@
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.net.http.HttpResponse.BodyHandler;
import java.net.http.HttpResponse.BodySubscriber;
import java.net.http.HttpResponse.BodySubscribers;
import java.net.http.HttpResponse.ResponseInfo;
import java.net.http.HttpTimeoutException;
import java.nio.ByteBuffer;
import java.time.Duration;
import java.util.Collections;
import java.util.List;
import java.util.Locale;
import java.util.Set;
import java.util.TreeSet;
Expand All @@ -37,6 +42,8 @@
import java.util.concurrent.Executor;
import java.util.concurrent.Flow;
import java.util.concurrent.TimeUnit;
import java.util.zip.GZIPInputStream;
import java.util.zip.InflaterInputStream;

import org.jspecify.annotations.Nullable;

Expand All @@ -59,6 +66,8 @@ class JdkClientHttpRequest extends AbstractStreamingClientHttpRequest {

private static final Set<String> DISALLOWED_HEADERS = disallowedHeaders();

private static final List<String> ALLOWED_ENCODINGS = List.of("gzip", "deflate");


private final HttpClient httpClient;

Expand All @@ -70,15 +79,18 @@ class JdkClientHttpRequest extends AbstractStreamingClientHttpRequest {

private final @Nullable Duration timeout;

private final boolean compressionEnabled;


public JdkClientHttpRequest(HttpClient httpClient, URI uri, HttpMethod method, Executor executor,
@Nullable Duration readTimeout) {
@Nullable Duration readTimeout, boolean compressionEnabled) {

this.httpClient = httpClient;
this.uri = uri;
this.method = method;
this.executor = executor;
this.timeout = readTimeout;
this.compressionEnabled = compressionEnabled;
}


Expand All @@ -98,7 +110,7 @@ protected ClientHttpResponse executeInternal(HttpHeaders headers, @Nullable Body
CompletableFuture<HttpResponse<InputStream>> responseFuture = null;
try {
HttpRequest request = buildRequest(headers, body);
responseFuture = this.httpClient.sendAsync(request, HttpResponse.BodyHandlers.ofInputStream());
responseFuture = this.httpClient.sendAsync(request, new DecompressingBodyHandler());

if (this.timeout != null) {
TimeoutHandler timeoutHandler = new TimeoutHandler(responseFuture, this.timeout);
Expand Down Expand Up @@ -141,6 +153,15 @@ else if (cause instanceof IOException ioEx) {
private HttpRequest buildRequest(HttpHeaders headers, @Nullable Body body) {
HttpRequest.Builder builder = HttpRequest.newBuilder().uri(this.uri);

// When compression is enabled and valid encoding is absent, we add gzip as standard encoding
if (this.compressionEnabled) {
if (headers.containsHeader(HttpHeaders.ACCEPT_ENCODING) &&
!ALLOWED_ENCODINGS.contains(headers.getFirst(HttpHeaders.ACCEPT_ENCODING))) {
headers.remove(HttpHeaders.ACCEPT_ENCODING);
}
headers.add(HttpHeaders.ACCEPT_ENCODING, "gzip");
}

headers.forEach((headerName, headerValues) -> {
if (!DISALLOWED_HEADERS.contains(headerName.toLowerCase(Locale.ROOT))) {
for (String headerValue : headerValues) {
Expand Down Expand Up @@ -226,7 +247,7 @@ public ByteBuffer map(byte[] b, int off, int len) {
/**
* Temporary workaround to use instead of {@link HttpRequest.Builder#timeout(Duration)}
* until <a href="https://bugs.openjdk.org/browse/JDK-8258397">JDK-8258397</a>
* is fixed. Essentially, create a future wiht a timeout handler, and use it
* is fixed. Essentially, create a future with a timeout handler, and use it
* to close the response.
* @see <a href="https://mail.openjdk.org/pipermail/net-dev/2021-October/016672.html">OpenJDK discussion thread</a>
*/
Expand Down Expand Up @@ -269,4 +290,39 @@ public void close() throws IOException {
}
}

/**
* Custom BodyHandler that checks the Content-Encoding header and applies the appropriate decompression algorithm.
* Supports Gzip and Deflate encoded responses.
*/
public static final class DecompressingBodyHandler implements BodyHandler<InputStream> {

@Override
public BodySubscriber<InputStream> apply(ResponseInfo responseInfo) {
String contentEncoding = responseInfo.headers().firstValue(HttpHeaders.CONTENT_ENCODING).orElse("");
if (contentEncoding.equalsIgnoreCase("gzip")) {
// If the content is gzipped, wrap the InputStream with a GZIPInputStream
return BodySubscribers.mapping(
BodySubscribers.ofInputStream(),
(InputStream is) -> {
try {
return new GZIPInputStream(is);
}
catch (IOException ex) {
throw new UncheckedIOException(ex); // Propagate IOExceptions
}
});
}
else if (contentEncoding.equalsIgnoreCase("deflate")) {
// If the content is encoded using deflate, wrap the InputStream with a InflaterInputStream
return BodySubscribers.mapping(
BodySubscribers.ofInputStream(),
InflaterInputStream::new);
}
else {
// Otherwise, return a standard InputStream BodySubscriber
return BodySubscribers.ofInputStream();
}
}
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,8 @@ public class JdkClientHttpRequestFactory implements ClientHttpRequestFactory {

private @Nullable Duration readTimeout;

private boolean compressionEnabled;


/**
* Create a new instance of the {@code JdkClientHttpRequestFactory}
Expand Down Expand Up @@ -96,10 +98,18 @@ public void setReadTimeout(Duration readTimeout) {
this.readTimeout = readTimeout;
}

/**
* Sets custom {@link BodyHandler} that can handle gzip encoded {@link HttpClient}'s response.
* @param compressionEnabled to enable compression by default for all {@link HttpClient}'s requests.
*/
public void setCompressionEnabled(boolean compressionEnabled) {
this.compressionEnabled = compressionEnabled;
}


@Override
public ClientHttpRequest createRequest(URI uri, HttpMethod httpMethod) throws IOException {
return new JdkClientHttpRequest(this.httpClient, uri, httpMethod, this.executor, this.readTimeout);
return new JdkClientHttpRequest(this.httpClient, uri, httpMethod, this.executor, this.readTimeout, this.compressionEnabled);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,14 @@
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;

import org.springframework.http.HttpHeaders;
import org.springframework.util.StringUtils;

import java.io.ByteArrayOutputStream;
import java.nio.charset.StandardCharsets;
import java.util.zip.DeflaterOutputStream;
import java.util.zip.GZIPOutputStream;

import static org.assertj.core.api.Assertions.assertThat;

/**
Expand Down Expand Up @@ -106,6 +112,26 @@ else if(request.getTarget().startsWith("/header/")) {
String headerName = request.getTarget().replace("/header/","");
return new MockResponse.Builder().body(headerName + ":" + request.getHeaders().get(headerName)).code(200).build();
}
else if(request.getTarget().startsWith("/compress/")) {
String encoding = request.getTarget().replace("/compress/","");
ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
if (encoding.equals("gzip")) {
try(GZIPOutputStream gzipOutputStream = new GZIPOutputStream(byteArrayOutputStream)) {
gzipOutputStream.write("Test Payload".getBytes());
gzipOutputStream.flush();
}
}
else if(encoding.equals("deflate")) {
try(DeflaterOutputStream deflaterOutputStream = new DeflaterOutputStream(byteArrayOutputStream)) {
deflaterOutputStream.write("Test Payload".getBytes());
deflaterOutputStream.flush();
}
} else {
byteArrayOutputStream.write("Test Payload".getBytes());
}
return new MockResponse.Builder().body(byteArrayOutputStream.toString(StandardCharsets.ISO_8859_1))
.code(200).setHeader(HttpHeaders.CONTENT_ENCODING, encoding).build();
}
return new MockResponse.Builder().code(404).build();
}
catch (Throwable ex) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,44 @@ void deleteRequestWithBody() throws Exception {
}
}

@Test
void compressionDisabled() throws IOException {
URI uri = URI.create(baseUrl + "/compress/");
ClientHttpRequest request = this.factory.createRequest(uri, HttpMethod.GET);
try (ClientHttpResponse response = request.execute()) {
assertThat(response.getStatusCode()).as("Invalid response status").isEqualTo(HttpStatus.OK);
assertThat(StreamUtils.copyToString(response.getBody(), StandardCharsets.ISO_8859_1))
.as("Invalid request body").isEqualTo("Test Payload");
}
}

@Test
void compressionGzip() throws IOException {
URI uri = URI.create(baseUrl + "/compress/gzip");
JdkClientHttpRequestFactory requestFactory = (JdkClientHttpRequestFactory) this.factory;
requestFactory.setCompressionEnabled(true);
ClientHttpRequest request = requestFactory.createRequest(uri, HttpMethod.GET);

try (ClientHttpResponse response = request.execute()) {
assertThat(response.getStatusCode()).as("Invalid response status").isEqualTo(HttpStatus.OK);
assertThat(StreamUtils.copyToString(response.getBody(), StandardCharsets.ISO_8859_1))
.as("Invalid request body").isEqualTo("Test Payload");
}
}

@Test
void compressionDeflate() throws IOException {
URI uri = URI.create(baseUrl + "/compress/deflate");
JdkClientHttpRequestFactory requestFactory = (JdkClientHttpRequestFactory) this.factory;
requestFactory.setCompressionEnabled(true);
ClientHttpRequest request = requestFactory.createRequest(uri, HttpMethod.GET);
try (ClientHttpResponse response = request.execute()) {
assertThat(response.getStatusCode()).as("Invalid response status").isEqualTo(HttpStatus.OK);
assertThat(StreamUtils.copyToString(response.getBody(), StandardCharsets.ISO_8859_1))
.as("Invalid request body").isEqualTo("Test Payload");
}
}

@Test // gh-34971
@EnabledForJreRange(min = JRE.JAVA_19) // behavior fixed in Java 19
void requestContentLengthHeaderWhenNoBody() throws Exception {
Expand Down