Add uploadSnowballObjects() method. (#1277)

balamurugana · web-flow · commit 57e34b7ed273 · 2022-01-12T19:30:19.000-08:00
Signed-off-by: Bala.FA &lt;bala@minio.io&gt;
diff --git a/api/src/main/java/io/minio/MinioClient.java b/api/src/main/java/io/minio/MinioClient.java
@@ -51,6 +51,9 @@
 import io.minio.messages.SseConfiguration;
 import io.minio.messages.Tags;
 import io.minio.messages.VersioningConfiguration;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.OutputStream;
@@ -64,6 +67,7 @@
 import java.nio.file.StandardOpenOption;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
+import java.util.Date;
 import java.util.Iterator;
 import java.util.LinkedList;
 import java.util.List;
@@ -73,6 +77,9 @@
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
 import okhttp3.Response;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
+import org.xerial.snappy.SnappyFramedOutputStream;
 
 /**
  * Simple Storage Service (aka S3) client to perform bucket and object operations.
@@ -2591,6 +2598,131 @@ public void deleteObjectTags(DeleteObjectTagsArgs args)
     executeDelete(args, null, queryParams);
   }
 
+  /**
+   * Uploads multiple objects in a single put call. It is done by creating intermediate TAR file
+   * optionally compressed which is uploaded to S3 service.
+   *
+   * <pre>Example:{@code
+   * // Upload snowball objects.
+   * List<SnowballObject> objects = new ArrayList<SnowballObject>();
+   * objects.add(
+   *     new SnowballObject(
+   *         "my-object-one",
+   *         new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8)),
+   *         5,
+   *         null));
+   * objects.add(
+   *     new SnowballObject(
+   *         "my-object-two",
+   *         new ByteArrayInputStream("java".getBytes(StandardCharsets.UTF_8)),
+   *         4,
+   *         null));
+   * minioClient.uploadSnowballObjects(
+   *     UploadSnowballObjectsArgs.builder().bucket("my-bucketname").objects(objects).build());
+   * }</pre>
+   *
+   * @param args {@link UploadSnowballObjectsArgs} object.
+   * @throws ErrorResponseException thrown to indicate S3 service returned an error response.
+   * @throws InsufficientDataException thrown to indicate not enough data available in InputStream.
+   * @throws InternalException thrown to indicate internal library error.
+   * @throws InvalidKeyException thrown to indicate missing of HMAC SHA-256 library.
+   * @throws InvalidResponseException thrown to indicate S3 service returned invalid or no error
+   *     response.
+   * @throws IOException thrown to indicate I/O error on S3 operation.
+   * @throws NoSuchAlgorithmException thrown to indicate missing of MD5 or SHA-256 digest library.
+   * @throws XmlParserException thrown to indicate XML parsing error.
+   */
+  public ObjectWriteResponse uploadSnowballObjects(UploadSnowballObjectsArgs args)
+      throws ErrorResponseException, InsufficientDataException, InternalException,
+          InvalidKeyException, InvalidResponseException, IOException, NoSuchAlgorithmException,
+          ServerException, XmlParserException {
+    checkArgs(args);
+
+    FileOutputStream fos = null;
+    BufferedOutputStream bos = null;
+    SnappyFramedOutputStream sos = null;
+    ByteArrayOutputStream baos = null;
+    TarArchiveOutputStream tarOutputStream = null;
+
+    try {
+      OutputStream os = null;
+      if (args.stagingFilename() != null) {
+        fos = new FileOutputStream(args.stagingFilename());
+        bos = new BufferedOutputStream(fos);
+        os = bos;
+      } else {
+        baos = new ByteArrayOutputStream();
+        os = baos;
+      }
+
+      if (args.compression()) {
+        sos = new SnappyFramedOutputStream(os);
+        os = sos;
+      }
+
+      tarOutputStream = new TarArchiveOutputStream(os);
+      for (SnowballObject object : args.objects()) {
+        if (object.filename() != null) {
+          Path filePath = Paths.get(object.filename());
+          TarArchiveEntry entry = new TarArchiveEntry(filePath.toFile(), object.name());
+          tarOutputStream.putArchiveEntry(entry);
+          Files.copy(filePath, tarOutputStream);
+        } else {
+          TarArchiveEntry entry = new TarArchiveEntry(object.name());
+          if (object.modificationTime() != null) {
+            entry.setModTime(Date.from(object.modificationTime().toInstant()));
+          }
+          entry.setSize(object.size());
+          tarOutputStream.putArchiveEntry(entry);
+          ByteStreams.copy(object.stream(), tarOutputStream);
+        }
+        tarOutputStream.closeArchiveEntry();
+      }
+      tarOutputStream.finish();
+    } finally {
+      if (tarOutputStream != null) tarOutputStream.flush();
+      if (sos != null) sos.flush();
+      if (bos != null) bos.flush();
+      if (fos != null) fos.flush();
+      if (tarOutputStream != null) tarOutputStream.close();
+      if (sos != null) sos.close();
+      if (bos != null) bos.close();
+      if (fos != null) fos.close();
+    }
+
+    Multimap<String, String> headers = newMultimap(args.extraHeaders());
+    headers.putAll(args.genHeaders());
+    headers.put("X-Amz-Meta-Snowball-Auto-Extract", "true");
+
+    if (args.stagingFilename() == null) {
+      byte[] data = baos.toByteArray();
+      return putObject(
+          args.bucket(),
+          args.region(),
+          args.object(),
+          data,
+          data.length,
+          headers,
+          args.extraQueryParams());
+    }
+
+    long length = Paths.get(args.stagingFilename()).toFile().length();
+    if (length > ObjectWriteArgs.MAX_OBJECT_SIZE) {
+      throw new IllegalArgumentException(
+          "tarball size " + length + " is more than maximum allowed 5TiB");
+    }
+    try (RandomAccessFile file = new RandomAccessFile(args.stagingFilename(), "r")) {
+      return putObject(
+          args.bucket(),
+          args.region(),
+          args.object(),
+          file,
+          length,
+          headers,
+          args.extraQueryParams());
+    }
+  }
+
   public static Builder builder() {
     return new Builder();
   }
diff --git a/api/src/main/java/io/minio/SnowballObject.java b/api/src/main/java/io/minio/SnowballObject.java
@@ -0,0 +1,74 @@
+/*
+ * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio;
+
+import java.io.InputStream;
+import java.time.ZonedDateTime;
+import java.util.Objects;
+import javax.annotation.Nonnull;
+import javax.annotation.Nullable;
+
+/** Single object entry for {@link UploadSnowballObjectsArgs#objects}. */
+public class SnowballObject {
+  private String name;
+  private InputStream stream;
+  private long size;
+  private ZonedDateTime modificationTime;
+  private String filename;
+
+  public SnowballObject(
+      @Nonnull String name,
+      @Nonnull InputStream stream,
+      long size,
+      @Nullable ZonedDateTime modificationTime) {
+    if (name == null || name.isEmpty()) throw new IllegalArgumentException("name must be provided");
+    this.name = name.startsWith("/") ? name.substring(1) : name;
+    this.stream = Objects.requireNonNull(stream, "stream must not be null");
+    if (size < 0) throw new IllegalArgumentException("size cannot be negative value");
+    this.size = size;
+    this.modificationTime = modificationTime;
+  }
+
+  public SnowballObject(@Nonnull String name, @Nonnull String filename) {
+    if (name == null || name.isEmpty()) throw new IllegalArgumentException("name must be provided");
+    this.name = name.startsWith("/") ? name.substring(1) : name;
+    if (filename == null || filename.isEmpty()) {
+      throw new IllegalArgumentException("filename must be provided");
+    }
+    this.filename = filename;
+  }
+
+  public String name() {
+    return this.name;
+  }
+
+  public InputStream stream() {
+    return this.stream;
+  }
+
+  public long size() {
+    return this.size;
+  }
+
+  public String filename() {
+    return this.filename;
+  }
+
+  public ZonedDateTime modificationTime() {
+    return this.modificationTime;
+  }
+}
diff --git a/api/src/main/java/io/minio/UploadSnowballObjectsArgs.java b/api/src/main/java/io/minio/UploadSnowballObjectsArgs.java
@@ -0,0 +1,96 @@
+/*
+ * MinIO Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2021 MinIO, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio;
+
+import java.security.SecureRandom;
+import java.util.Objects;
+import java.util.Random;
+
+/** Argument class of {@link MinioClient#uploadSnowballObjects}. */
+public class UploadSnowballObjectsArgs extends ObjectWriteArgs {
+  private static final Random random = new Random(new SecureRandom().nextLong());
+
+  private Iterable<SnowballObject> objects;
+  private String stagingFilename;
+  private boolean compression;
+
+  public Iterable<SnowballObject> objects() {
+    return this.objects;
+  }
+
+  public String stagingFilename() {
+    return stagingFilename;
+  }
+
+  public boolean compression() {
+    return compression;
+  }
+
+  public static Builder builder() {
+    return new Builder();
+  }
+
+  /** Argument builder of {@link UploadSnowballObjectsArgs}. */
+  public static final class Builder
+      extends ObjectWriteArgs.Builder<Builder, UploadSnowballObjectsArgs> {
+    private void validateObjects(Iterable<SnowballObject> objects) {
+      validateNotNull(objects, "objects");
+    }
+
+    @Override
+    protected void validate(UploadSnowballObjectsArgs args) {
+      args.objectName = "snowball." + random.nextLong() + ".tar";
+      validateObjects(args.objects);
+      super.validate(args);
+    }
+
+    public Builder objects(Iterable<SnowballObject> objects) {
+      validateObjects(objects);
+      operations.add(args -> args.objects = objects);
+      return this;
+    }
+
+    public Builder stagingFilename(String stagingFilename) {
+      if (stagingFilename != null && stagingFilename.isEmpty()) {
+        throw new IllegalArgumentException("staging filename must not be empty");
+      }
+      operations.add(args -> args.stagingFilename = stagingFilename);
+      return this;
+    }
+
+    public Builder compression(boolean compression) {
+      operations.add(args -> args.compression = compression);
+      return this;
+    }
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (this == o) return true;
+    if (!(o instanceof UploadSnowballObjectsArgs)) return false;
+    if (!super.equals(o)) return false;
+    UploadSnowballObjectsArgs that = (UploadSnowballObjectsArgs) o;
+    return Objects.equals(objects, that.objects)
+        && Objects.equals(stagingFilename, that.stagingFilename)
+        && compression == that.compression;
+  }
+
+  @Override
+  public int hashCode() {
+    return Objects.hash(super.hashCode(), objects, stagingFilename, compression);
+  }
+}
diff --git a/build.gradle b/build.gradle
@@ -58,6 +58,8 @@ subprojects {
         compile "com.fasterxml.jackson.core:jackson-core:2.11.2"
         compile "com.fasterxml.jackson.core:jackson-databind:2.11.2"
         compile "org.bouncycastle:bcprov-jdk15on:1.69"
+        compile 'org.apache.commons:commons-compress:1.21'
+        compile "org.xerial.snappy:snappy-java:1.1.8.4"
         compileOnly "com.github.spotbugs:spotbugs-annotations:4.1.2"
 
         testImplementation "com.squareup.okhttp3:mockwebserver:4.8.1"
diff --git a/docs/API.md b/docs/API.md
@@ -45,7 +45,7 @@ MinioClient minioClient =
 | [`makeBucket`](#makeBucket)                                       | [`setObjectTags`](#setObjectTags)                       |
 | [`removeBucket`](#removeBucket)                                   | [`statObject`](#statObject)                             |
 | [`setBucketEncryption`](#setBucketEncryption)                     | [`uploadObject`](#uploadObject)                         |
-| [`setBucketLifecycle`](#setBucketLifecycle)                       |                                                         |
+| [`setBucketLifecycle`](#setBucketLifecycle)                       | [`uploadSnowballObjects`](#uploadSnowballObjects)       |
 | [`setBucketNotification`](#setBucketNotification)                 |                                                         |
 | [`setBucketPolicy`](#setBucketPolicy)                             |                                                         |
 | [`setBucketReplication`](#setBucketReplication)                   |                                                         |
@@ -1543,6 +1543,36 @@ minioClient.uploadObject(
         .build());
 ```
 
+<a name="uploadSnowballObjects"></a>
+### uploadSnowballObjects(UploadSnowballObjectsArgs args)
+`public void uploadSnowballObjects(UploadSnowballObjectsArgs args)` _[[Javadoc]](http://minio.github.io/minio-java/io/minio/MinioClient.html#uploadSnowballObjects-io.minio.UploadSnowballObjectsArgs-)_
+
+Uploads multiple objects in a single put call. It is done by creating intermediate TAR file optionally compressed which is uploaded to S3 service.
+
+__Parameters__
+| Parameter | Type                          | Description |
+|:----------|:------------------------------|:------------|
+| ``args``  | _[UploadSnowballObjectsArgs]_ | Arguments.  |
+
+__Example__
+```java
+List<SnowballObject> objects = new ArrayList<SnowballObject>();
+objects.add(
+    new SnowballObject(
+        "my-object-one",
+        new ByteArrayInputStream("hello".getBytes(StandardCharsets.UTF_8)),
+        5,
+        null));
+objects.add(
+    new SnowballObject(
+        "my-object-two",
+        new ByteArrayInputStream("java".getBytes(StandardCharsets.UTF_8)),
+        4,
+        null));
+minioClient.uploadSnowballObjects(
+    UploadSnowballObjectsArgs.builder().bucket("my-bucketname").objects(objects).build());
+```
+
 <a name="removeObject"></a>
 ### removeObject(RemoveObjectArgs args)
 `public void removeObject(RemoveObjectArgs args)` _[[Javadoc]](http://minio.github.io/minio-java/io/minio/MinioClient.html#removeObject-io.minio.RemoveObjectArgs-)_
@@ -1830,6 +1860,7 @@ ObjectStat objectStat =
 [CopyObjectArgs]: http://minio.github.io/minio-java/io/minio/CopyObjectArgs.html
 [PutObjectArgs]: http://minio.github.io/minio-java/io/minio/PutObjectArgs.html
 [UploadObjectArgs]: http://minio.github.io/minio-java/io/minio/UploadObjectArgs.html
+[UploadSnowballObjectsArgs]: http://minio.github.io/minio-java/io/minio/UploadSnowballObjectsArgs.html
 [ComposeObjectArgs]: http://minio.github.io/minio-java/io/minio/ComposeObjectArgs.html
 [ObjectWriteResponse]: http://minio.github.io/minio-java/io/minio/ObjectWriteResponse.html
 [ListBucketsArgs]: http://minio.github.io/minio-java/io/minio/ListBucketsArgs.html
diff --git a/examples/UploadSnowballObjects.java b/examples/UploadSnowballObjects.java
diff --git a/functional/FunctionalTest.java b/functional/FunctionalTest.java
