api: add new API to delete multiple objects. (#553)

* api: add new API to delete multiple objects.

This patch adds new API

public Iterable<Result<DeleteError>> removeObject(final String bucketName,
    final Iterable<String> objectNames)

which uses "Delete Multiple Objects" ReST API
(http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html).

Fixes #467

* Add example and docs.

Author: balamurugana

README.md

@@ -137,6 +137,7 @@ The full API Reference is available here.
 * [GetObject.Java](https://github.com/minio/minio-java/tree/master/examples/GetObject.java)
 * [GetPartialObject.java](https://github.com/minio/minio-java/tree/master/examples/GetPartialObject.java)
 * [RemoveObject.java](https://github.com/minio/minio-java/tree/master/examples/RemoveObject.java)
+* [RemoveObjects.java](https://github.com/minio/minio-java/tree/master/examples/RemoveObjects.java)
 * [StatObject.java](https://github.com/minio/minio-java/tree/master/examples/StatObject.java)
 
 #### Full Examples: Presigned Operations

api/src/main/java/io/minio/MinioClient.java

@@ -42,6 +42,10 @@
 import io.minio.messages.CompleteMultipartUpload;
 import io.minio.messages.CopyObjectResult;
 import io.minio.messages.CreateBucketConfiguration;
+import io.minio.messages.DeleteError;
+import io.minio.messages.DeleteObject;
+import io.minio.messages.DeleteRequest;
+import io.minio.messages.DeleteResult;
 import io.minio.messages.ErrorResponse;
 import io.minio.messages.InitiateMultipartUploadResult;
 import io.minio.messages.Item;
@@ -1884,6 +1888,160 @@ public void removeObject(String bucketName, String objectName)
   }
 
 
+  private List<DeleteError> removeObject(String bucketName, List<DeleteObject> objectList)
+    throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException, IOException,
+           InvalidKeyException, NoResponseException, XmlPullParserException, ErrorResponseException,
+           InternalException {
+    Map<String,String> queryParamMap = new HashMap<>();
+    queryParamMap.put("delete", "");
+
+    DeleteRequest request = new DeleteRequest(objectList);
+    HttpResponse response = executePost(bucketName, null, null, queryParamMap, request);
+
+    String bodyContent = "";
+    // Use scanner to read entire body stream to string.
+    Scanner scanner = new Scanner(response.body().charStream());
+    try {
+      scanner.useDelimiter("\\A");
+      if (scanner.hasNext()) {
+        bodyContent = scanner.next();
+      }
+    } finally {
+      response.body().close();
+      scanner.close();
+    }
+
+    List<DeleteError> errorList = null;
+
+    bodyContent = bodyContent.trim();
+    // Check if body content is <Error> message.
+    DeleteError error = new DeleteError(new StringReader(bodyContent));
+    if (error.code() != null) {
+      // As it is <Error> message, add to error list.
+      errorList = new LinkedList<DeleteError>();
+      errorList.add(error);
+    } else {
+      // As it is not <Error> message, parse it as <DeleteResult> message.
+      DeleteResult result = new DeleteResult(new StringReader(bodyContent));
+      errorList = result.errorList();
+    }
+
+    return errorList;
+  }
+
+
+  /**
+   * Removes multiple objects from a bucket.
+   *
+   * </p><b>Example:</b><br>
+   * <pre>{@code // Create object list for removal.
+   * List<String> objectNames = new LinkedList<String>();
+   * objectNames.add("my-objectname1");
+   * objectNames.add("my-objectname2");
+   * objectNames.add("my-objectname3");
+   * for (Result<DeleteError> errorResult: minioClient.removeObject("my-bucketname", objectNames)) {
+   *     DeleteError error = errorResult.get();
+   *     System.out.println("Failed to remove '" + error.objectName() + "'. Error:" + error.message());
+   * } }</pre>
+   *
+   * @param bucketName Bucket name.
+   * @param objectNames List of Object names in the bucket.
+   */
+  public Iterable<Result<DeleteError>> removeObject(final String bucketName, final Iterable<String> objectNames) {
+    return new Iterable<Result<DeleteError>>() {
+      @Override
+      public Iterator<Result<DeleteError>> iterator() {
+        return new Iterator<Result<DeleteError>>() {
+          private Result<DeleteError> error;
+          private Iterator<DeleteError> errorIterator;
+          private boolean completed = false;
+
+          private synchronized void populate() {
+            List<DeleteError> errorList = null;
+            try {
+              List<DeleteObject> objectList = new LinkedList<DeleteObject>();
+              int i = 0;
+              for (String objectName: objectNames) {
+                objectList.add(new DeleteObject(objectName));
+                i++;
+                // Maximum 1000 objects are allowed in a request
+                if (i == 1000) {
+                  break;
+                }
+              }
+
+              if (i == 0) {
+                return;
+              }
+
+              errorList = removeObject(bucketName, objectList);
+            } catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException
+                     | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException
+                     | InternalException e) {
+              this.error = new Result<>(null, e);
+            } finally {
+              if (errorList != null) {
+                this.errorIterator = errorList.iterator();
+              } else {
+                this.errorIterator = new LinkedList<DeleteError>().iterator();
+              }
+            }
+          }
+
+          @Override
+          public boolean hasNext() {
+            if (this.completed) {
+              return false;
+            }
+
+            if (this.error == null && this.errorIterator == null) {
+              populate();
+            }
+
+            if (this.error != null) {
+              return true;
+            }
+
+            if (this.errorIterator.hasNext()) {
+              return true;
+            }
+
+            this.completed = true;
+            return false;
+          }
+
+          @Override
+          public Result<DeleteError> next() {
+            if (this.completed) {
+              throw new NoSuchElementException();
+            }
+
+            if (this.error == null && this.errorIterator == null) {
+              populate();
+            }
+
+            if (this.error != null) {
+              this.completed = true;
+              return this.error;
+            }
+
+            if (this.errorIterator.hasNext()) {
+              return new Result<>(this.errorIterator.next(), null);
+            }
+
+            this.completed = true;
+            throw new NoSuchElementException();
+          }
+
+          @Override
+          public void remove() {
+            throw new UnsupportedOperationException();
+          }
+        };
+      }
+    };
+  }
+
   /**
    * Lists object information in given bucket.
    *

api/src/main/java/io/minio/messages/DeleteError.java

@@ -0,0 +1,46 @@
+/*
+ * Minio Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio.messages;
+
+import java.io.IOException;
+import java.io.Reader;
+
+import org.xmlpull.v1.XmlPullParserException;
+
+
+/**
+ * Helper class to parse Amazon AWS S3 error response XML.
+ */
+@SuppressWarnings("unused")
+public class DeleteError extends ErrorResponse {
+  /**
+   * Constructs a new ErrorResponse object by reading given reader stream.
+   */
+  public DeleteError() throws XmlPullParserException {
+    super();
+    super.name = "Error";
+  }
+
+
+  /**
+   * Constructs a new ErrorResponse object by reading given reader stream.
+   */
+  public DeleteError(Reader reader) throws IOException, XmlPullParserException {
+    this();
+    this.parseXml(reader);
+  }
+}

api/src/main/java/io/minio/messages/DeleteObject.java

@@ -0,0 +1,51 @@
+/*
+ * Minio Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio.messages;
+
+import org.xmlpull.v1.XmlPullParserException;
+
+import com.google.api.client.util.Key;
+
+
+/**
+ * Helper class to create Amazon AWS S3 request XML containing object information for Multiple object deletion.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressFBWarnings(value = "URF_UNREAD_FIELD")
+public class DeleteObject extends XmlEntity {
+  @Key("Key")
+  private String name;
+  @Key("VersionId")
+  private String versionId;
+
+
+  public DeleteObject(String name) throws XmlPullParserException {
+    this(name, null);
+  }
+
+
+  /**
+   * Constructs new delete object for given name and version ID.
+   *
+   */
+  public DeleteObject(String name, String versionId) throws XmlPullParserException {
+    super();
+    super.name = "Object";
+
+    this.name = name;
+    this.versionId = versionId;
+  }
+}

api/src/main/java/io/minio/messages/DeleteRequest.java

@@ -0,0 +1,53 @@
+/*
+ * Minio Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio.messages;
+
+import java.util.List;
+
+import org.xmlpull.v1.XmlPullParserException;
+
+import com.google.api.client.util.Key;
+
+
+/**
+ * Helper class to create Amazon AWS S3 request XML containing information for Multiple object deletion.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressFBWarnings(value = "URF_UNREAD_FIELD")
+public class DeleteRequest extends XmlEntity {
+  @Key("Quiet")
+  private boolean quiet;
+  @Key("Object")
+  private List<DeleteObject> objectList;
+
+
+  public DeleteRequest(List<DeleteObject> objectList) throws XmlPullParserException {
+    this(objectList, true);
+  }
+
+
+  /**
+   * Constructs new delete request for given object list and quiet flag.
+   */
+  public DeleteRequest(List<DeleteObject> objectList, boolean quiet) throws XmlPullParserException {
+    super();
+    super.name = "Delete";
+    super.namespaceDictionary.set("", "http://s3.amazonaws.com/doc/2006-03-01/");
+
+    this.objectList = objectList;
+    this.quiet = quiet;
+  }
+}

api/src/main/java/io/minio/messages/DeleteResult.java

@@ -0,0 +1,58 @@
+/*
+ * Minio Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2017 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package io.minio.messages;
+
+import java.io.IOException;
+import java.io.Reader;
+import java.util.LinkedList;
+import java.util.List;
+
+import org.xmlpull.v1.XmlPullParserException;
+
+import com.google.api.client.util.Key;
+
+
+/**
+ * Helper class to create Amazon AWS S3 request XML containing information for Multiple object deletion.
+ */
+@SuppressWarnings({"SameParameterValue", "unused"})
+public class DeleteResult extends XmlEntity {
+  @Key("Deleted")
+  private List<DeletedObject> objectList = new LinkedList<>();
+  @Key("Error")
+  private List<DeleteError> errorList = new LinkedList<>();
+
+
+  /**
+   * Constructs new delete result by parsing content on given reader.
+   */
+  public DeleteResult(Reader reader) throws IOException, XmlPullParserException {
+    super();
+    super.name = "DeleteResult";
+    this.parseXml(reader);
+  }
+
+
+  public List<DeletedObject> objectList() {
+    return objectList;
+  }
+
+
+  public List<DeleteError> errorList() {
+    return errorList;
+  }
+}