Handle reading more than 2GiB part data correctly. (#1205)

Fixes #1204

Signed-off-by: Bala.FA <bala.gluster@gmail.com>

Author: balamurugana

api/src/main/java/io/minio/ByteBufferStream.java

@@ -0,0 +1,33 @@
+/*
+ * MinIO Java SDK for Amazon S3 Compatible Cloud Storage,
+ * (C) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.InputStream;
+
+/** ByteArrayOutputStream exposes underneath buffer as input stream. */
+class ByteBufferStream extends ByteArrayOutputStream {
+  public ByteBufferStream() {
+    super();
+  }
+
+  public InputStream inputStream() {
+    return new ByteArrayInputStream(this.buf, 0, this.count);
+  }
+}

api/src/main/java/io/minio/Digest.java

@@ -25,51 +25,48 @@
 import java.nio.charset.StandardCharsets;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
 import java.util.Locale;
 
 /** Various global static functions used. */
 public class Digest {
+  // MD5 hash of zero length byte array.
+  public static final String ZERO_MD5_HASH = "1B2M2Y8AsgTpgAmY7PhCfg==";
+  // SHA-256 hash of zero length byte array.
+  public static final String ZERO_SHA256_HASH =
+      "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+
   /** Private constructor. */
   private Digest() {}
 
-  /** Returns SHA-256 hash of given string. */
-  public static String sha256Hash(String string) throws NoSuchAlgorithmException {
-    byte[] data = string.getBytes(StandardCharsets.UTF_8);
-    MessageDigest sha256Digest = MessageDigest.getInstance("SHA-256");
-    sha256Digest.update((byte[]) data, 0, data.length);
-    return BaseEncoding.base16().encode(sha256Digest.digest()).toLowerCase(Locale.US);
+  /** Returns MD5 hash of byte array. */
+  public static String md5Hash(byte[] data, int length) throws NoSuchAlgorithmException {
+    MessageDigest md5Digest = MessageDigest.getInstance("MD5");
+    md5Digest.update(data, 0, length);
+    return Base64.getEncoder().encodeToString(md5Digest.digest());
   }
 
-  /**
-   * Returns SHA-256 hash of given data and it's length.
-   *
-   * @param data must be {@link RandomAccessFile}, {@link BufferedInputStream} or byte array.
-   * @param len length of data to be read for hash calculation.
-   */
-  public static String sha256Hash(Object data, int len)
-      throws NoSuchAlgorithmException, IOException, InsufficientDataException, InternalException {
+  /** Returns SHA-256 hash of byte array. */
+  public static String sha256Hash(byte[] data, int length) throws NoSuchAlgorithmException {
     MessageDigest sha256Digest = MessageDigest.getInstance("SHA-256");
-
-    if (data instanceof BufferedInputStream || data instanceof RandomAccessFile) {
-      updateDigests(data, len, sha256Digest, null);
-    } else if (data instanceof byte[]) {
-      sha256Digest.update((byte[]) data, 0, len);
-    } else {
-      throw new InternalException(
-          "Unknown data source to calculate SHA-256 hash. This should not happen, "
-              + "please report this issue at https://github.com/minio/minio-java/issues",
-          null);
-    }
-
+    sha256Digest.update((byte[]) data, 0, length);
     return BaseEncoding.base16().encode(sha256Digest.digest()).toLowerCase(Locale.US);
   }
 
+  /** Returns SHA-256 hash of given string. */
+  public static String sha256Hash(String string) throws NoSuchAlgorithmException {
+    byte[] data = string.getBytes(StandardCharsets.UTF_8);
+    return sha256Hash(data, data.length);
+  }
+
   /**
    * Returns SHA-256 and MD5 hashes of given data and it's length.
    *
    * @param data must be {@link RandomAccessFile}, {@link BufferedInputStream} or byte array.
    * @param len length of data to be read for hash calculation.
+   * @deprecated This method is no longer supported.
    */
+  @Deprecated
   public static String[] sha256Md5Hashes(Object data, int len)
       throws NoSuchAlgorithmException, IOException, InsufficientDataException, InternalException {
     MessageDigest sha256Digest = MessageDigest.getInstance("SHA-256");
@@ -93,30 +90,6 @@ public static String[] sha256Md5Hashes(Object data, int len)
     };
   }
 
-  /**
-   * Returns MD5 hash of given data and it's length.
-   *
-   * @param data must be {@link RandomAccessFile}, {@link BufferedInputStream} or byte array.
-   * @param len length of data to be read for hash calculation.
-   */
-  public static String md5Hash(Object data, int len)
-      throws NoSuchAlgorithmException, IOException, InsufficientDataException, InternalException {
-    MessageDigest md5Digest = MessageDigest.getInstance("MD5");
-
-    if (data instanceof BufferedInputStream || data instanceof RandomAccessFile) {
-      updateDigests(data, len, null, md5Digest);
-    } else if (data instanceof byte[]) {
-      md5Digest.update((byte[]) data, 0, len);
-    } else {
-      throw new InternalException(
-          "Unknown data source to calculate MD5 hash. This should not happen, "
-              + "please report this issue at https://github.com/minio/minio-java/issues",
-          null);
-    }
-
-    return BaseEncoding.base64().encode(md5Digest.digest());
-  }
-
   /** Updated MessageDigest with bytes read from file and stream. */
   private static int updateDigests(
       Object inputStream, int len, MessageDigest sha256Digest, MessageDigest md5Digest)

api/src/main/java/io/minio/HttpRequestBody.java

@@ -16,32 +16,20 @@
 
 package io.minio;
 
-import java.io.BufferedInputStream;
 import java.io.IOException;
-import java.io.RandomAccessFile;
-import java.nio.channels.Channels;
 import okhttp3.MediaType;
 import okhttp3.RequestBody;
 import okio.BufferedSink;
-import okio.Okio;
 
 /** RequestBody that wraps a single data object. */
 class HttpRequestBody extends RequestBody {
-  private RandomAccessFile file = null;
-  private BufferedInputStream stream = null;
-  private byte[] bytes = null;
-  private int length = -1;
-  private String contentType = null;
+  private PartSource partSource;
+  private byte[] bytes;
+  private int length;
+  private String contentType;
 
-  HttpRequestBody(final RandomAccessFile file, final int length, final String contentType) {
-    this.file = file;
-    this.length = length;
-    this.contentType = contentType;
-  }
-
-  HttpRequestBody(final BufferedInputStream stream, final int length, final String contentType) {
-    this.stream = stream;
-    this.length = length;
+  HttpRequestBody(final PartSource partSource, final String contentType) {
+    this.partSource = partSource;
     this.contentType = contentType;
   }
 
@@ -54,28 +42,19 @@ class HttpRequestBody extends RequestBody {
   @Override
   public MediaType contentType() {
     MediaType mediaType = null;
-
-    if (contentType != null) {
-      mediaType = MediaType.parse(contentType);
-    }
-    if (mediaType == null) {
-      mediaType = MediaType.parse("application/octet-stream");
-    }
-
-    return mediaType;
+    if (contentType != null) mediaType = MediaType.parse(contentType);
+    return (mediaType == null) ? MediaType.parse("application/octet-stream") : mediaType;
   }
 
   @Override
   public long contentLength() {
-    return length;
+    return (partSource != null) ? partSource.size() : length;
   }
 
   @Override
   public void writeTo(BufferedSink sink) throws IOException {
-    if (file != null) {
-      sink.write(Okio.source(Channels.newInputStream(file.getChannel())), length);
-    } else if (stream != null) {
-      sink.write(Okio.source(stream), length);
+    if (partSource != null) {
+      sink.write(partSource.source(), partSource.size());
     } else {
       sink.write(bytes, 0, length);
     }

api/src/main/java/io/minio/PartReader.java

@@ -0,0 +1,183 @@
+/*
+ * MinIO Java SDK for Amazon S3 Compatible Cloud Storage,
+ * (C) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio;
+
+import com.google.common.io.BaseEncoding;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.RandomAccessFile;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Base64;
+import java.util.Locale;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+
+/** PartReader reads part data from file or input stream sequentially and returns PartSource. */
+class PartReader {
+  private static final long CHUNK_SIZE = Integer.MAX_VALUE;
+
+  private byte[] buf16k = new byte[16384]; // 16KiB buffer for optimization.
+
+  private RandomAccessFile file;
+  private InputStream stream;
+
+  private long objectSize;
+  private long partSize;
+  private int partCount;
+
+  private int partNumber;
+  private long totalDataRead;
+
+  private ByteBufferStream[] buffers;
+  private byte[] oneByte = null;
+  boolean eof;
+
+  private PartReader(long objectSize, long partSize, int partCount) {
+    this.objectSize = objectSize;
+    this.partSize = partSize;
+    this.partCount = partCount;
+
+    long bufferCount = partSize / CHUNK_SIZE;
+    if ((partSize - (bufferCount * CHUNK_SIZE)) > 0) bufferCount++;
+    if (bufferCount == 0) bufferCount++;
+
+    this.buffers = new ByteBufferStream[(int) bufferCount];
+  }
+
+  public PartReader(@Nonnull RandomAccessFile file, long objectSize, long partSize, int partCount) {
+    this(objectSize, partSize, partCount);
+    this.file = Objects.requireNonNull(file, "file must not be null");
+    if (this.objectSize < 0) throw new IllegalArgumentException("object size must be provided");
+  }
+
+  public PartReader(@Nonnull InputStream stream, long objectSize, long partSize, int partCount) {
+    this(objectSize, partSize, partCount);
+    this.stream = Objects.requireNonNull(stream, "stream must not be null");
+    for (int i = 0; i < this.buffers.length; i++) this.buffers[i] = new ByteBufferStream();
+  }
+
+  private long readStreamChunk(
+      ByteBufferStream buffer, long size, MessageDigest md5, MessageDigest sha256)
+      throws IOException {
+    long totalBytesRead = 0;
+
+    if (this.oneByte != null) {
+      buffer.write(this.oneByte);
+      md5.update(this.oneByte);
+      if (sha256 != null) sha256.update(this.oneByte);
+      totalBytesRead++;
+      this.oneByte = null;
+    }
+
+    while (totalBytesRead < size) {
+      long bytesToRead = size - totalBytesRead;
+      if (bytesToRead > this.buf16k.length) bytesToRead = this.buf16k.length;
+      int bytesRead = this.stream.read(this.buf16k, 0, (int) bytesToRead);
+      this.eof = (bytesRead < 0);
+      if (this.eof) {
+        if (this.objectSize < 0) break;
+        throw new IOException("unexpected EOF");
+      }
+      buffer.write(this.buf16k, 0, bytesRead);
+      md5.update(this.buf16k, 0, bytesRead);
+      if (sha256 != null) sha256.update(this.buf16k, 0, bytesRead);
+      totalBytesRead += bytesRead;
+    }
+
+    return totalBytesRead;
+  }
+
+  private long readStream(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
+    long count = size / CHUNK_SIZE;
+    long lastChunkSize = size - (count * CHUNK_SIZE);
+    if (lastChunkSize > 0) {
+      count++;
+    } else {
+      lastChunkSize = CHUNK_SIZE;
+    }
+
+    long totalBytesRead = 0;
+    for (int i = 0; i < buffers.length; i++) buffers[i].reset();
+    for (long i = 1; i <= count && !this.eof; i++) {
+      long chunkSize = (i != count) ? CHUNK_SIZE : lastChunkSize;
+      long bytesRead = this.readStreamChunk(buffers[(int) (i - 1)], chunkSize, md5, sha256);
+      totalBytesRead += bytesRead;
+    }
+
+    if (!this.eof && this.objectSize < 0) {
+      this.oneByte = new byte[1];
+      this.eof = this.stream.read(this.oneByte) < 0;
+    }
+
+    return totalBytesRead;
+  }
+
+  private long readFile(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
+    long position = this.file.getFilePointer();
+    long totalBytesRead = 0;
+
+    while (totalBytesRead < size) {
+      long bytesToRead = size - totalBytesRead;
+      if (bytesToRead > this.buf16k.length) bytesToRead = this.buf16k.length;
+      int bytesRead = this.file.read(this.buf16k, 0, (int) bytesToRead);
+      if (bytesRead < 0) throw new IOException("unexpected EOF");
+      md5.update(this.buf16k, 0, bytesRead);
+      if (sha256 != null) sha256.update(this.buf16k, 0, bytesRead);
+      totalBytesRead += bytesRead;
+    }
+
+    this.file.seek(position);
+    return totalBytesRead;
+  }
+
+  private long read(long size, MessageDigest md5, MessageDigest sha256) throws IOException {
+    return (this.file != null) ? readFile(size, md5, sha256) : readStream(size, md5, sha256);
+  }
+
+  public PartSource getPart(boolean computeSha256) throws NoSuchAlgorithmException, IOException {
+    if (this.partNumber == this.partCount) return null;
+
+    this.partNumber++;
+
+    MessageDigest md5 = MessageDigest.getInstance("MD5");
+    MessageDigest sha256 = computeSha256 ? MessageDigest.getInstance("SHA-256") : null;
+
+    long partSize = this.partSize;
+    if (this.partNumber == this.partCount) partSize = this.objectSize - this.totalDataRead;
+    long bytesRead = this.read(partSize, md5, sha256);
+    this.totalDataRead += bytesRead;
+    if (this.objectSize < 0 && this.eof) this.partCount = this.partNumber;
+
+    String md5Hash = Base64.getEncoder().encodeToString(md5.digest());
+    String sha256Hash = null;
+    if (computeSha256) {
+      sha256Hash = BaseEncoding.base16().encode(sha256.digest()).toLowerCase(Locale.US);
+    }
+
+    if (this.file != null) {
+      return new PartSource(this.partNumber, this.file, bytesRead, md5Hash, sha256Hash);
+    }
+
+    return new PartSource(this.partNumber, this.buffers, bytesRead, md5Hash, sha256Hash);
+  }
+
+  public int partCount() {
+    return this.partCount;
+  }
+}