Skip to content

Commit a11dd4c

Browse files
abelhegedusabelhegedus
committed
make sure structure crawling does not crawl elements
session details are written to file, printed to console and deleted after successful login
1 parent f16a17a commit a11dd4c

File tree

5 files changed

+76
-72
lines changed

5 files changed

+76
-72
lines changed

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,13 @@
99

1010
Running the jar without arguments prints out the usage instructions.
1111

12+
- Running the crawler without workspace, resource, branch and revision will result in crawling the repository structure of the given scope
13+
- Running the crawler with workspace, resource, branch and revision will result in crawling the model content of the given revision
14+
15+
### Repository structure of OpenSE Cookbook workspace on twc.openmbee.org
16+
17+
`-S twc.openmbee.org -P 8111 -ssl -W 9c368adc-10cc-45d9-bec6-27aedc80e68b -C 2000 -u openmbeeguest -pw guest`
18+
1219
### OpenSE Cookbook model on twc.openmbee.org
1320

1421
`-S twc.openmbee.org -P 8111 -ssl -W 9c368adc-10cc-45d9-bec6-27aedc80e68b -R c6bede89-cd5e-487b-aca8-a4f384370759 -B 29110c0f-bdc1-4294-ae24-9fd608629cac -REV 350 -C 2000 -u openmbeeguest -pw guest`

‎build.gradle‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
plugins {
2-
id 'org.jetbrains.kotlin.jvm' version '1.3.50'
2+
id 'org.jetbrains.kotlin.jvm' version '1.3.60'
33
id 'io.vertx.vertx-plugin' version '0.9.0'
44
}
55

‎src/main/kotlin/Crawler.kt‎

Lines changed: 8 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@ import io.vertx.core.json.Json
1111
import io.vertx.core.json.JsonObject
1212
import java.io.File
1313

14-
15-
private val CHUNK_SIZE = "chunkSize"
14+
const val CHUNK_SIZE = "chunkSize"
1615

1716
fun main(args: Array<String>) {
1817

@@ -128,11 +127,9 @@ fun main(args: Array<String>) {
128127
twcMap[CHUNK_SIZE] = chunkSize
129128
println("Chunk size is $chunkSize")
130129

131-
if (instanceNum != null) {
132-
println("Instance number set to $instanceNum")
133-
if (instanceNum < 1) {
134-
error("Number of Instances should be at least 1.")
135-
}
130+
println("Instance number set to $instanceNum")
131+
if (instanceNum < 1) {
132+
error("Number of Instances should be at least 1.")
136133
}
137134
if (workspaceId != null) {
138135
println("Workspace ID set to $workspaceId")
@@ -200,7 +197,7 @@ fun main(args: Array<String>) {
200197

201198
val restVerticle = RESTVerticle()
202199

203-
var options = DeploymentOptions().setWorker(true).setHa(true).setInstances(instanceNum).setWorkerPoolSize(32)
200+
val options = DeploymentOptions().setWorker(true).setHa(true).setInstances(instanceNum).setWorkerPoolSize(32)
204201

205202
twcMap["flag"] = 0
206203
vertx.deployVerticle(restVerticle.javaClass.name, options) { deploy ->
@@ -217,9 +214,9 @@ fun main(args: Array<String>) {
217214
usr,
218215
pswd
219216
)
220-
) { deploy ->
221-
if (deploy.failed()) {
222-
error("Deploy failed: ${deploy.cause().message}\n${deploy.cause().printStackTrace()}")
217+
) {
218+
if (it.failed()) {
219+
error("Deploy failed: ${it.cause().message}\n${it.cause().printStackTrace()}")
223220
}
224221
}
225222
}

‎src/main/kotlin/verticles/MainVerticle.kt‎

Lines changed: 58 additions & 59 deletions
Original file line numberDiff line numberDiff line change
@@ -16,29 +16,29 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
1616

1717
val eb = vertx.eventBus()
1818
vertx.setPeriodic(1000) {
19-
vertx.sharedData().getCounter(QUERIES) { res ->
20-
if (res.succeeded()) {
21-
val counter = res.result()
22-
counter.get { res ->
23-
queries = res.result().toInt()
19+
vertx.sharedData().getCounter(QUERIES) { queriesR ->
20+
if (queriesR.succeeded()) {
21+
val queriesC = queriesR.result()
22+
queriesC.get { queriesV ->
23+
this.queries = queriesV.result().toInt()
2424

25-
vertx.sharedData().getCounter("sum") { res ->
26-
if (res.succeeded()) {
27-
val counter = res.result()
28-
counter.get { get ->
29-
if (get.succeeded()) {
30-
sum = get.result().toInt()
25+
vertx.sharedData().getCounter("sum") { sumR ->
26+
if (sumR.succeeded()) {
27+
val sumC = sumR.result()
28+
sumC.get { sumV ->
29+
if (sumV.succeeded()) {
30+
this.sum = sumV.result().toInt()
3131

32-
vertx.sharedData().getCounter("number") { res ->
33-
if (res.succeeded()) {
34-
val counter = res.result()
35-
counter.get { get ->
36-
if (get.succeeded()) {
37-
number = get.result().toInt()
38-
println("Total: ${queries}/${sum}/${++s} query/response/sec | Now: $number elem | AvgSpeed: ${sum / (s)} elem/sec")
39-
counter.compareAndSet(number.toLong(), 0, {})
32+
vertx.sharedData().getCounter("number") { numR ->
33+
if (numR.succeeded()) {
34+
val numC = numR.result()
35+
numC.get { numV ->
36+
if (numV.succeeded()) {
37+
number = numV.result().toInt()
38+
println("Total: ${this.queries}/${this.sum}/${++s} query/response/sec | Now: $number elem | AvgSpeed: ${this.sum / (s)} elem/sec")
39+
numC.compareAndSet(number.toLong(), 0, {})
4040

41-
if (queries != 0 && sum != 0 && queries == sum) {
41+
if (this.queries != 0 && this.sum != 0 && this.queries == this.sum) {
4242
eb.send(
4343
TWCVERT_ADDRESS,
4444
JsonObject.mapFrom(
@@ -52,13 +52,13 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
5252
}
5353
}
5454
} else {
55-
error("Counter: queries not available.")
55+
error("Counter: queriesR not available.")
5656
}
5757
}
5858
}
5959
}
6060
} else {
61-
error("Counter: queries not available.")
61+
error("Counter: queriesR not available.")
6262
}
6363
}
6464

@@ -175,8 +175,8 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
175175

176176
}
177177
REPO -> {
178-
// println("Received Repository")
179178
val repo = JsonObject(messageData.obj as Map<String, Any>).mapTo(Repo::class.java)
179+
println("Received workspace list $repo")
180180
repo.workspaces.forEach { ws ->
181181
val id = JsonObject(ws as Map<String, Any>).getString("@id")
182182
eb.send(
@@ -194,8 +194,8 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
194194
}
195195
}
196196
WORKSPACE -> {
197-
// println("Received Workspace")
198197
val workspace = JsonObject(messageData.obj as Map<String, Any>).mapTo(Workspace::class.java)
198+
println("Received workspace content for $workspace")
199199
val workspaceId = workspace.id
200200

201201
workspace.resources.forEach { res ->
@@ -219,8 +219,8 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
219219
}
220220
}
221221
RESOURCE -> {
222-
// println("Received Resource")
223222
val resource = JsonObject(messageData.obj as Map<String, Any>).mapTo(Resource::class.java)
223+
println("Received resource content for $resource")
224224
val resourceId = resource.id
225225
val workspaceId = resource.workspace_id
226226
resource.branches.forEach { branch ->
@@ -249,48 +249,47 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
249249

250250
}
251251
BRANCH -> {
252-
// println("Received Branch")
253252
val branch = JsonObject(messageData.obj as Map<String, Any>).mapTo(Branch::class.java)
253+
println("Received branch content for $branch")
254254
val branchId = branch.id
255255
val resourceId = branch.resource_id
256256
val workspaceId = branch.workspace_id
257-
val inputRevision = twcMap[REVISION]
258-
if (inputRevision == null) {
259-
twcMap[REVISION] = branch.revisions.maxBy { it as Int }
260-
}
261257

262258
branch.revisions.forEach { rev ->
263259
val revId = (rev as Int)
264-
vertx.eventBus().send(
265-
TWCVERT_ADDRESS, JsonObject.mapFrom(
266-
Message(
267-
GET_ROOT_ELEMENT_IDS,
268-
JsonObject()
269-
.put(
270-
WORKSPACE_ID,
271-
workspaceId
272-
)
273-
.put(
274-
RESOURCE_ID,
275-
resourceId
276-
)
277-
.put(
278-
BRANCH_ID,
279-
branchId
280-
)
281-
.put(
282-
REVISION_ID,
283-
revId
284-
)
260+
val inputRevision = twcMap[REVISION]
261+
if (revId == inputRevision) {
262+
vertx.eventBus().send(
263+
TWCVERT_ADDRESS, JsonObject.mapFrom(
264+
Message(
265+
GET_ROOT_ELEMENT_IDS,
266+
JsonObject()
267+
.put(
268+
WORKSPACE_ID,
269+
workspaceId
270+
)
271+
.put(
272+
RESOURCE_ID,
273+
resourceId
274+
)
275+
.put(
276+
BRANCH_ID,
277+
branchId
278+
)
279+
.put(
280+
REVISION_ID,
281+
revId
282+
)
283+
)
285284
)
286285
)
287-
)
286+
}
288287
}
289288

290289
}
291290
REVISION -> {
292-
// println("Received Revision")
293291
val revision = JsonObject(messageData.obj as Map<String, Any>).mapTo(Revision::class.java)
292+
println("Received revision content for $revision")
294293
val revisionId = revision.id
295294
val branchId = revision.branch_id
296295
val resourceId = revision.resource_id
@@ -365,12 +364,12 @@ class MainVerticle(val usr: String, val pswd: String) : AbstractVerticle() {
365364
}
366365
ELEMENT -> {
367366
// println("Received Element")
368-
val element = JsonObject(messageData.obj as Map<String, Any>).mapTo(Element::class.java)
369-
val revisionId = element.revision_id
370-
val branchId = element.branch_id
371-
val resourceId = element.resource_id
372-
val workspaceId = element.workspace_id
373-
element.elements.forEach { element ->
367+
val elementRequest = JsonObject(messageData.obj as Map<String, Any>).mapTo(Element::class.java)
368+
val revisionId = elementRequest.revision_id
369+
val branchId = elementRequest.branch_id
370+
val resourceId = elementRequest.resource_id
371+
val workspaceId = elementRequest.workspace_id
372+
elementRequest.elements.forEach { element ->
374373
val elementId = JsonObject(element as Map<String, Any>).getString("@id")
375374

376375
vertx.eventBus().send(

‎src/main/kotlin/verticles/RESTVerticle.kt‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -308,6 +308,7 @@ class RESTVerticle() : AbstractVerticle() {
308308
}
309309

310310
private fun getRevisions(client: WebClient, twcMap: LocalMap<Any, Any>, obj: JsonObject) {
311+
queryPrepared(1)
311312
val workspaceId = obj.getString(WORKSPACE_ID)
312313
val resourceId = obj.getString(RESOURCE_ID)
313314
val branchId = obj.getString(BRANCH_ID)
@@ -321,7 +322,7 @@ class RESTVerticle() : AbstractVerticle() {
321322
.send { ar ->
322323
if (ar.succeeded()) {
323324
if (ar.result().statusCode() == 200) {
324-
325+
queryCompleted(1)
325326
val data = ar.result().bodyAsJsonArray()
326327
val branch = Branch(
327328
branchId,

0 commit comments

Comments
 (0)