Update crawler with additional options

Author: abelhegedus

com.incquerylabs.vhci.modelaccess.twc.rest/build.gradle

@@ -1,33 +1,28 @@
-group 'com.incquerylabs'
-version '1.0-SNAPSHOT'
-
-buildscript {
-    ext.kotlin_version = '1.1.4-3'
-    ext.vertx_version = '3.4.2'
-
-    repositories {
-        mavenCentral()
-    }
-    dependencies {
-        classpath "org.jetbrains.kotlin:kotlin-gradle-plugin:$kotlin_version"
-    }
+plugins {
+    id 'org.jetbrains.kotlin.jvm' version '1.1.50'
+    id 'io.vertx.vertx-plugin' version '0.0.3'
 }
 
-apply plugin: 'kotlin'
+group 'com.incquerylabs'
+version '1.0-SNAPSHOT'
 
 repositories {
     mavenCentral()
 }
 
 dependencies {
-    compile "org.jetbrains.kotlin:kotlin-stdlib-jre8:$kotlin_version"
-    compile "io.vertx:vertx-web-client:$vertx_version"
-    compile "io.vertx:vertx-web:$vertx_version"
+    compile 'org.jetbrains.kotlin:kotlin-stdlib-jre8'
+    compile 'io.vertx:vertx-web'
+    compile 'io.vertx:vertx-web-client'
 }
 
 compileKotlin {
-    kotlinOptions.jvmTarget = "1.8"
+    kotlinOptions.jvmTarget = '1.8'
 }
 compileTestKotlin {
-    kotlinOptions.jvmTarget = "1.8"
+    kotlinOptions.jvmTarget = '1.8'
+}
+
+vertx {
+    mainVerticle = 'com.incquerylabs.vhci.modelaccess.twc.rest.verticles.MainVerticle'
 }

com.incquerylabs.vhci.modelaccess.twc.rest/src/main/kotlin/com/incquerylabs/vhci/modelaccess/twc/rest/Crawler.kt

@@ -10,9 +10,6 @@ import io.vertx.core.DeploymentOptions
 import io.vertx.core.cli.Argument
 import io.vertx.core.cli.CLI
 import io.vertx.core.cli.Option
-import io.vertx.core.shareddata.LocalMap
-import io.vertx.ext.web.client.WebClient
-import io.vertx.ext.web.client.WebClientOptions
 import java.io.File
 
 
@@ -72,12 +69,17 @@ fun main(args: Array<String>) {
                     Option()
                         .setLongName("debug")
                         .setShortName("D")
-                        .setDescription("Set number of RESTVerticle instances. Default:16")
+                        .setDescription("Enable debug logging. Default: false")
+                        .setFlag(true),
+                    Option()
+                        .setLongName("requestSingleElement")
+                        .setShortName("RSE")
+                        .setDescription("Request elements one-by-one. Default: false")
                         .setFlag(true),
                     Option()
                         .setLongName(CHUNK_SIZE)
                         .setShortName("C")
-                        .setDescription("Set the size of chunks to use when crawling elements. Default -1 to disable chunks")
+                        .setDescription("Set the size of chunks to use when crawling elements. Default: -1 to disable chunks")
                         .setDefaultValue("-1")
             ))
 
@@ -105,13 +107,20 @@ fun main(args: Array<String>) {
     val revision = commandLine.getOptionValue<String>("revision")
     val chunkSize = commandLine.getOptionValue<String>(CHUNK_SIZE).toInt()
     val debug = commandLine.isFlagEnabled("debug")
+    val requestSingleElement = commandLine.isFlagEnabled("requestSingleElement")
 
     twcMap.put("debug", debug)
     if(debug) {
         println("Debug mode is enabled")
     }
-    println("Chunk size is $chunkSize")
+
+    twcMap.put("requestSingleElement", requestSingleElement)
+    if(requestSingleElement) {
+        println("Request single elements mode is enabled")
+    }
+
     twcMap.put(CHUNK_SIZE, chunkSize)
+    println("Chunk size is $chunkSize")
 
     if(instanceNum!=null){
         println("Instance number set to $instanceNum")

com.incquerylabs.vhci.modelaccess.twc.rest/src/main/kotlin/com/incquerylabs/vhci/modelaccess/twc/rest/data/Data.kt

@@ -1,17 +1,18 @@
 package com.incquerylabs.vhci.modelaccess.twc.rest.data
 
 import io.vertx.core.json.JsonArray
+import io.vertx.core.json.JsonObject
 
-data class Message(val event:String, val obj:Any)
-data class User(val username:String, val password:String)
-data class Server(val path:String, val port:Int)
-data class Repo(val id: String, val workspaces:JsonArray)
-data class Workspace(val id:String, val resources:JsonArray)
-data class Resource(val id:String, val workspace_id:String, val branches:JsonArray)
-data class Branch(val id:String,val resource_id:String ,val workspace_id:String,val revisions:JsonArray)
-data class Revision(val id:Int, val branch_id:String, val resource_id:String ,val workspace_id:String, val elements:JsonArray)
-data class Element(val id:String, val revision_id:Int, val branch_id:String, val resource_id:String ,val workspace_id:String, val elements:JsonArray)
-data class Elements(val revision_id:Int, val branch_id:String, val resource_id:String ,val workspace_id:String, val elements:JsonArray)
+data class Message(val event: String = "", val obj: Any = "")
+data class User(val username: String, val password: String)
+data class Server(val path: String, val port: Int)
+data class Repo(val id: String = "", val workspaces: List<Any?> = emptyList())
+data class Workspace(val id: String = "", val resources: List<Any?> = emptyList())
+data class Resource(val id: String = "", val workspace_id: String = "", val branches: List<Any?> = emptyList())
+data class Branch(val id: String = "", val resource_id: String = "", val workspace_id: String = "", val revisions: List<Any?> = emptyList())
+data class Revision(val id: Int = -1, val branch_id: String = "", val resource_id: String = "", val workspace_id: String = "", val elements: List<Any?> = emptyList())
+data class Element(val id: String = "", val revision_id: Int = -1, val branch_id: String = "", val resource_id: String = "", val workspace_id: String = "", val elements: List<Any?> = emptyList())
+data class Elements(val revision_id: Int = -1, val branch_id: String = "", val resource_id: String = "", val workspace_id: String = "", val elements: List<Any?> = emptyList())
 
 object DataConstants {
     const val REPO = "repo"