api: add list object version 2 support and set to default. (#552)

Now ListObjects() API uses ReST API List Objects Version 2
(http://docs.aws.amazon.com/AmazonS3/latest/API/v2-RESTBucketGET.html)
as a default.

Added new method
listObjects(String bucketName, String prefix, boolean recursive,
            boolean useVersion1)

which can be used for ReST API List Objects Version 1 by passing true
to useVersion1 flag.

Fixes #413

Author: balamurugana

api/src/main/java/io/minio/MinioClient.java

@@ -51,6 +51,7 @@
 import io.minio.messages.Item;
 import io.minio.messages.ListAllMyBucketsResult;
 import io.minio.messages.ListBucketResult;
+import io.minio.messages.ListBucketResultV1;
 import io.minio.messages.ListMultipartUploadsResult;
 import io.minio.messages.ListPartsResult;
 import io.minio.messages.Part;
@@ -2089,19 +2090,222 @@ public Iterable<Result<Item>> listObjects(final String bucketName, final String
    *
    * @see #listObjects(String bucketName)
    * @see #listObjects(String bucketName, String prefix)
+   * @see #listObjects(String bucketName, String prefix, boolean recursive, boolean useVersion1)
    */
   public Iterable<Result<Item>> listObjects(final String bucketName, final String prefix, final boolean recursive) {
+    return listObjects(bucketName, prefix, recursive, false);
+  }
+
+
+  /**
+   * Lists object information as {@code Iterable<Result><Item>>} in given bucket, prefix, recursive flag and S3 API
+   * version to use.
+   *
+   * </p><b>Example:</b><br>
+   * <pre>{@code Iterable<Result<Item>> myObjects = minioClient.listObjects("my-bucketname", "my-object-prefix", true,
+   *                                    false);
+   * for (Result<Item> result : myObjects) {
+   *   Item item = result.get();
+   *   System.out.println(item.lastModified() + ", " + item.size() + ", " + item.objectName());
+   * } }</pre>
+   *
+   * @param bucketName Bucket name.
+   * @param prefix     Prefix string.  List objects whose name starts with `prefix`.
+   * @param recursive when false, emulates a directory structure where each listing returned is either a full object
+   *                  or part of the object's key up to the first '/'. All objects wit the same prefix up to the first
+   *                  '/' will be merged into one entry.
+   * @param useVersion1 If set, Amazon AWS S3 List Object V1 is used, else List Object V2 is used as default.
+   *
+   * @return an iterator of Result Items.
+   *
+   * @see #listObjects(String bucketName)
+   * @see #listObjects(String bucketName, String prefix)
+   * @see #listObjects(String bucketName, String prefix, boolean recursive)
+   */
+  public Iterable<Result<Item>> listObjects(final String bucketName, final String prefix, final boolean recursive,
+                                            final boolean useVersion1) {
+    if (useVersion1) {
+      return listObjectsV1(bucketName, prefix, recursive);
+    }
+
+    return listObjectsV2(bucketName, prefix, recursive);
+  }
+
+  
+  private Iterable<Result<Item>> listObjectsV2(final String bucketName, final String prefix, final boolean recursive) {
     return new Iterable<Result<Item>>() {
       @Override
       public Iterator<Result<Item>> iterator() {
         return new Iterator<Result<Item>>() {
-          private String lastObjectName;
           private ListBucketResult listBucketResult;
           private Result<Item> error;
           private Iterator<Item> itemIterator;
           private Iterator<Prefix> prefixIterator;
           private boolean completed = false;
 
+          private synchronized void populate() {
+            String delimiter = "/";
+            if (recursive) {
+              delimiter = null;
+            }
+
+            String continuationToken = null;
+            if (this.listBucketResult != null && delimiter != null) {
+              continuationToken = listBucketResult.nextContinuationToken();
+            }
+
+            this.listBucketResult = null;
+            this.itemIterator = null;
+            this.prefixIterator = null;
+
+            try {
+              this.listBucketResult = listObjectsV2(bucketName, continuationToken, prefix, delimiter);
+            } catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException
+                     | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException
+                     | InternalException e) {
+              this.error = new Result<>(null, e);
+            } finally {
+              if (this.listBucketResult != null) {
+                this.itemIterator = this.listBucketResult.contents().iterator();
+                this.prefixIterator = this.listBucketResult.commonPrefixes().iterator();
+              } else {
+                this.itemIterator = new LinkedList<Item>().iterator();
+                this.prefixIterator = new LinkedList<Prefix>().iterator();
+              }
+            }
+          }
+
+          @Override
+          public boolean hasNext() {
+            if (this.completed) {
+              return false;
+            }
+
+            if (this.error == null && this.itemIterator == null && this.prefixIterator == null) {
+              populate();
+            }
+
+            if (this.error == null && !this.itemIterator.hasNext() && !this.prefixIterator.hasNext()
+                && this.listBucketResult.isTruncated()) {
+              populate();
+            }
+
+            if (this.error != null) {
+              return true;
+            }
+
+            if (this.itemIterator.hasNext()) {
+              return true;
+            }
+
+            if (this.prefixIterator.hasNext()) {
+              return true;
+            }
+
+            this.completed = true;
+            return false;
+          }
+
+          @Override
+          public Result<Item> next() {
+            if (this.completed) {
+              throw new NoSuchElementException();
+            }
+
+            if (this.error == null && this.itemIterator == null && this.prefixIterator == null) {
+              populate();
+            }
+
+            if (this.error == null && !this.itemIterator.hasNext() && !this.prefixIterator.hasNext()
+                && this.listBucketResult.isTruncated()) {
+              populate();
+            }
+
+            if (this.error != null) {
+              this.completed = true;
+              return this.error;
+            }
+
+            if (this.itemIterator.hasNext()) {
+              Item item = this.itemIterator.next();
+              return new Result<>(item, null);
+            }
+
+            if (this.prefixIterator.hasNext()) {
+              Prefix prefix = this.prefixIterator.next();
+              Item item;
+              try {
+                item = new Item(prefix.prefix(), true);
+              } catch (XmlPullParserException e) {
+                // special case: ignore the error as we can't propagate the exception in next()
+                item = null;
+              }
+
+              return new Result<>(item, null);
+            }
+
+            this.completed = true;
+            throw new NoSuchElementException();
+          }
+
+          @Override
+          public void remove() {
+            throw new UnsupportedOperationException();
+          }
+        };
+      }
+    };
+  }
+
+
+  /**
+   * Returns {@link ListBucketResult} of given bucket, marker, prefix and delimiter.
+   *
+   * @param bucketName Bucket name.
+   * @param marker     Marker string.  List objects whose name is greater than `marker`.
+   * @param prefix     Prefix string.  List objects whose name starts with `prefix`.
+   * @param delimiter  delimiter string.  Group objects whose name contains `delimiter`.
+   */
+  private ListBucketResult listObjectsV2(String bucketName, String continuationToken, String prefix, String delimiter)
+    throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException, IOException,
+           InvalidKeyException, NoResponseException, XmlPullParserException, ErrorResponseException,
+           InternalException {
+    Map<String,String> queryParamMap = new HashMap<>();
+    queryParamMap.put("list-type", "2");
+
+    if (continuationToken != null) {
+      queryParamMap.put("continuation-token", continuationToken);
+    }
+
+    if (prefix != null) {
+      queryParamMap.put("prefix", prefix);
+    }
+
+    if (delimiter != null) {
+      queryParamMap.put("delimiter", delimiter);
+    }
+
+    HttpResponse response = executeGet(bucketName, null, null, queryParamMap);
+
+    ListBucketResult result = new ListBucketResult();
+    result.parseXml(response.body().charStream());
+    response.body().close();
+    return result;
+  }
+
+
+  private Iterable<Result<Item>> listObjectsV1(final String bucketName, final String prefix, final boolean recursive) {
+    return new Iterable<Result<Item>>() {
+      @Override
+      public Iterator<Result<Item>> iterator() {
+        return new Iterator<Result<Item>>() {
+          private String lastObjectName;
+          private ListBucketResultV1 listBucketResult;
+          private Result<Item> error;
+          private Iterator<Item> itemIterator;
+          private Iterator<Prefix> prefixIterator;
+          private boolean completed = false;
+
           private synchronized void populate() {
             String delimiter = "/";
             if (recursive) {
@@ -2122,7 +2326,7 @@ private synchronized void populate() {
             this.prefixIterator = null;
 
             try {
-              this.listBucketResult = listObjects(bucketName, marker, prefix, delimiter, null);
+              this.listBucketResult = listObjectsV1(bucketName, marker, prefix, delimiter);
             } catch (InvalidBucketNameException | NoSuchAlgorithmException | InsufficientDataException | IOException
                      | InvalidKeyException | NoResponseException | XmlPullParserException | ErrorResponseException
                      | InternalException e) {
@@ -2223,16 +2427,15 @@ public void remove() {
 
 
   /**
-   * Returns {@link ListBucketResult} of given bucket, marker, prefix, delimiter and maxKeys.
+   * Returns {@link ListBucketResultV1} of given bucket, marker, prefix and delimiter.
    *
    * @param bucketName Bucket name.
    * @param marker     Marker string.  List objects whose name is greater than `marker`.
    * @param prefix     Prefix string.  List objects whose name starts with `prefix`.
    * @param delimiter  delimiter string.  Group objects whose name contains `delimiter`.
    * @param maxKeys    Maximum number of entries to be returned.
    */
-  private ListBucketResult listObjects(String bucketName, String marker, String prefix, String delimiter,
-                                       Integer maxKeys)
+  private ListBucketResultV1 listObjectsV1(String bucketName, String marker, String prefix, String delimiter)
     throws InvalidBucketNameException, NoSuchAlgorithmException, InsufficientDataException, IOException,
            InvalidKeyException, NoResponseException, XmlPullParserException, ErrorResponseException,
            InternalException {
@@ -2250,13 +2453,9 @@ private ListBucketResult listObjects(String bucketName, String marker, String pr
       queryParamMap.put("delimiter", delimiter);
     }
 
-    if (maxKeys != null) {
-      queryParamMap.put("max-keys", maxKeys.toString());
-    }
-
     HttpResponse response = executeGet(bucketName, null, null, queryParamMap);
 
-    ListBucketResult result = new ListBucketResult();
+    ListBucketResultV1 result = new ListBucketResultV1();
     result.parseXml(response.body().charStream());
     response.body().close();
     return result;

api/src/main/java/io/minio/messages/ListBucketResult.java

@@ -1,5 +1,5 @@
 /*
- * Minio Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2015 Minio, Inc.
+ * Minio Java SDK for Amazon S3 Compatible Cloud Storage, (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,18 +25,22 @@
 
 
 /**
- * Helper class to parse Amazon AWS S3 response XML containing ListBucketResult information.
+ * Helper class to parse Amazon AWS S3 response XML containing ListBucketResult Version 2 information.
  */
 @SuppressWarnings({"SameParameterValue", "unused"})
 public class ListBucketResult extends XmlEntity {
   @Key("Name")
   private String name;
   @Key("Prefix")
   private String prefix;
-  @Key("Marker")
-  private String marker;
-  @Key("NextMarker")
-  private String nextMarker;
+  @Key("ContinuationToken")
+  private String continuationToken;
+  @Key("NextContinuationToken")
+  private String nextContinuationToken;
+  @Key("StartAfter")
+  private String startAfter;
+  @Key("KeyCount")
+  private String keyCount;
   @Key("MaxKeys")
   private int maxKeys;
   @Key("Delimiter")
@@ -55,14 +59,6 @@ public ListBucketResult() throws XmlPullParserException {
   }
 
 
-  /**
-   * Returns next marker.
-   */
-  public String nextMarker() {
-    return nextMarker;
-  }
-
-
   /**
    * Returns bucket name.
    */
@@ -80,10 +76,34 @@ public String prefix() {
 
 
   /**
-   * Returns marker.
+   * Returns continuation token.
+   */
+  public String continuationToken() {
+    return continuationToken;
+  }
+
+
+  /**
+   * Returns next continuation token.
+   */
+  public String nextContinuationToken() {
+    return nextContinuationToken;
+  }
+
+
+  /**
+   * Returns start after.
+   */
+  public String startAfter() {
+    return startAfter;
+  }
+
+
+  /**
+   * Returns key count.
    */
-  public String marker() {
-    return marker;
+  public String keyCount() {
+    return keyCount;
   }