Skip to content

Commit cf4680b

Browse files
author
Chris Wiechmann
committed
Changes for ES V8
1 parent 143ffbb commit cf4680b

26 files changed

+530
-1469
lines changed

‎api-builder-plugin-fn-elasticsearch/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
{
22
"name": "@axway-api-builder-ext/api-builder-plugin-fn-elasticsearch",
3-
"version": "2.1.2",
3+
"version": "3.0.0",
44
"description": "Integrate Elasticsearch into your API-Builder flow to combine search data for instance with other data available in your flow.",
55
"author": "Chris Wiechmann <cwiechmann@axway.com> (http://www.axway.com)",
66
"license": "Apache-2.0",

‎api-builder-plugin-fn-elasticsearch/src/actions/indices.js‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ async function indicesRollover(params, options) {
4949
logger.debug(`Rolling over all indicies starting with alias: ${alias}`);
5050
var foundAliases = await client.indices.getAlias({name: `${alias}*`}, { ignore: [404], maxRetries: 3 });
5151
// Based on the alias name we get all indicies
52-
for (const [key, value] of Object.entries(foundAliases.body)) {
52+
for (const [key, value] of Object.entries(foundAliases)) {
5353
for (const [aliasName, aliasSettings] of Object.entries(value.aliases)) {
5454
// Each can have only one write index, which we use to perform the rollover
5555
if(aliasSettings.is_write_index) {
@@ -121,6 +121,7 @@ async function indicesCreate(params, options) {
121121

122122
async function indicesExists(params, options) {
123123
const elasticSearchConfig = options.pluginConfig.elastic;
124+
debugger;
124125

125126
if (typeof elasticSearchConfig.node === 'undefined' && typeof elasticSearchConfig.nodes === 'undefined') {
126127
options.logger.error('Elasticsearch configuration is invalid: nodes or node is missing.');
@@ -140,7 +141,7 @@ async function indicesExists(params, options) {
140141
// This searches for indices only
141142
result = await client.indices.exists( params, { ignore: [404], maxRetries: 3 });
142143
}
143-
if(result.statusCode == 404) {
144+
if(!result) {
144145
return options.setOutput('notFound', result);
145146
}
146147

‎api-builder-plugin-fn-elasticsearch/src/actions/transform.js‎

Lines changed: 13 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -52,49 +52,51 @@ async function putTransform(params, options) {
5252
var actualTransform;
5353
var actualTransformId
5454
// Get all active (RUNNING ONLY) transforms with the given primary Transform-ID
55-
const allTransforms = await client.transform.getTransformStats({ transformId: `${params.transformId}*` }, { ignore: [404], maxRetries: 3 });
55+
debugger;
56+
const allTransforms = await client.transform.getTransformStats({ transform_id: `${params.transformId}*` }, { ignore: [404], maxRetries: 3, allowNoMatch: true });
5657
var runningTransforms = [];
57-
for (i = 0; i < allTransforms.body.transforms.length; i++) {
58-
const transform = allTransforms.body.transforms[i];
58+
for (i = 0; i < allTransforms.transforms.length; i++) {
59+
const transform = allTransforms.transforms[i];
5960
// Check if the Transform already exists, which means nothing to do
6061
if(transform.id==`${params.transformId}${idSuffix}`) {
6162
options.logger.info(`Transform found: ${params.transformId}${idSuffix} already exists with state: ${transform.state}. To update this transform, please provide an idSuffix (e.g. v2)`);
6263
if(startTransform && transform.state != "started" && transform.state != "indexing") {
6364
options.logger.info(`Existing transform: ${params.transformId}${idSuffix} is not running, going to start it.`);
64-
await client.transform.startTransform( {transformId: transform.id}, { ignore: [404], maxRetries: 3 });
65+
await client.transform.startTransform( {transform_id: transform.id}, { ignore: [404], maxRetries: 3 });
6566
}
6667
actualTransform = transform;
6768
} else {
6869
// Stop all other transforms
6970
if(transform.state == "started" || transform.state == "indexing") {
70-
await client.transform.stopTransform( {transformId: transform.id}, { ignore: [404], maxRetries: 3 });
71+
await client.transform.stopTransform( {transform_id: transform.id}, { ignore: [404], maxRetries: 3 });
7172
}
7273
}
7374
}
7475
if(actualTransform) return actualTransform;
7576
// Stop all running transforms, as we expect only one transform to run
7677
for (i = 0; i < runningTransforms.length; i++) {
7778
const runningTransform = runningTransforms[i];
78-
await client.transform.stopTransform( {transformId: runningTransform.id}, { ignore: [404], maxRetries: 3 });
79+
await client.transform.stopTransform( {transform_id: runningTransform.id}, { ignore: [404], maxRetries: 3 });
7980
}
8081

81-
params.transformId = `${params.transformId}${idSuffix}`;
82+
params.transform_id = `${params.transformId}${idSuffix}`;
83+
delete params.transformId; // Currently not supported by the JS-Library (See https://github.com/elastic/elasticsearch-js/issues/1645)
8284
try {
8385
var putTransformResult = await client.transform.putTransform( params, { ignore: [404], maxRetries: 3 });
8486
} catch (e) {
8587
if(e.meta.statusCode == 409) {
86-
throw new Error(`Error creating the transform. The transform id: \'${params.transformId}\' was used previously. This includes deleted transforms.`);
88+
throw new Error(`Error creating the transform. The transform id: \'${params.transform_id}\' was used previously. This includes deleted transforms.`);
8789
} else {
8890
throw e;
8991
}
9092
}
9193
if(startTransform) {
92-
options.logger.info(`Starting created transform with ID: ${params.transformId}`);
93-
await client.transform.startTransform( {transformId: params.transformId}, { ignore: [404], maxRetries: 3 });
94+
options.logger.info(`Starting created transform with ID: ${params.transform_id}`);
95+
await client.transform.startTransform( {transform_id: params.transform_id}, { ignore: [404], maxRetries: 3 });
9496
}
9597
if(deletePreviousTransform && actualTransformId) {
9698
options.logger.info(`Deleting previous transform with ID: ${actualTransformId}`);
97-
await client.transform.deleteTransform( {transformId: actualTransformId}, { ignore: [404], maxRetries: 3 });
99+
await client.transform.deleteTransform( {transform_id: actualTransformId}, { ignore: [404], maxRetries: 3 });
98100
}
99101
return putTransformResult;
100102
} catch (e) {

‎api-builder-plugin-fn-elasticsearch/src/flow-nodes.yml‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1041,13 +1041,13 @@ flow-nodes:
10411041
description: Indicates the specified index incl. aliases exist.
10421042
context: $.result
10431043
schema:
1044-
type: object
1044+
type: boolean
10451045
notFound:
10461046
name: Not found
10471047
description: Indicates one or more specified index aliases do not exist.
10481048
context: $.result
10491049
schema:
1050-
type: object
1050+
type: boolean
10511051
error:
10521052
name: Error
10531053
description: An unexpected error happened

‎api-builder-plugin-fn-elasticsearch/test/integration/integration-test.js‎

Lines changed: 21 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -55,15 +55,22 @@ describe('Integration tests', () => {
5555
});
5656

5757
describe('#Indices integration test', () => {
58-
5958
it('should fail if the given index-template does not exist', async () => {
6059
const inputParameter = { index: "my_index_to_be_created", indexTemplate: "this_template_is_missing" };
6160
const { value, output } = await flowNode.indicesCreate(inputParameter);
6261

6362
expect(value).to.be.instanceOf(Error)
6463
.and.to.have.property('message', "The index template: 'this_template_is_missing' is missing. Index wont be created.");
6564
expect(output).to.equal('error');
66-
});
65+
});
66+
67+
it('should return with not found for an unknown ALIAS', async () => {
68+
69+
const inputParameter = { name: "unknown-alias" };
70+
const { value, output } = await flowNode.indicesExists(inputParameter);
71+
72+
expect(output).to.equal('notFound');
73+
});
6774
});
6875

6976
describe('#Index mapping integration test', () => {
@@ -128,4 +135,16 @@ describe('Integration tests', () => {
128135
expect(output).to.equal('notFound');
129136
});
130137
});
138+
139+
describe('#Transform jobs integration test', () => {
140+
it('should create the transform not yet exists', async () => {
141+
const inputParameter = {
142+
transformId: 'traffic-summary-rollup-job',
143+
body: JSON.parse(fs.readFileSync('./test/unit/mock/transform/putTransformRequestBody.json')) };
144+
const { value, output } = await flowNode.putTransform(inputParameter);
145+
146+
expect(output).to.equal('next');
147+
});
148+
});
149+
131150
});

‎api-builder-plugin-fn-elasticsearch/test/unit/indexTemplate/indexTemplate-Test.js‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -80,12 +80,10 @@ describe('Template indices tests', () => {
8080
const { value, output } = await flowNode.putTemplate(inputParameter);
8181

8282
expect(output).to.equal('next');
83-
expect(value.statusCode).to.equal(200);
83+
expect(value.acknowledged).to.equal(true);
8484
expect(mockedFn.callCount).to.equals(1);
8585
// Validate all given parameters has been passed to the JS-Elastic client
8686
expect(mockedFn.lastCall.arg).to.deep.equals(inputParameter);
87-
// Make sure a body is returned
88-
expect(value.body).to.exist;
8987
});
9088

9189
it('should update index template if version number is higher / existing ILM policy should be taken over', async () => {
@@ -101,14 +99,13 @@ describe('Template indices tests', () => {
10199
};
102100
const { value, output } = await flowNode.putTemplate(inputParameter);
103101

102+
debugger;
104103
expect(output).to.equal('next');
105-
expect(value.statusCode).to.equal(200);
104+
expect(value.acknowledged).to.equal(true);
106105
expect(mockedPutTemplate.callCount).to.equals(1);
107106
expect(mockedPutTemplate.lastCall.arg.body.version).to.equals(2); // Make sure version 2 is given
108107
expect(mockedPutTemplate.lastCall.arg.body.aliases['apigw-traffic-summary']).to.exist; // Validate the existing mapping is returned
109108
expect(mockedPutTemplate.lastCall.arg.body.settings.index.lifecycle).to.deep.equals({name: "logstash-policy", rollover_alias: "apigw-traffic-summary"}); // Validate the existing mapping is returned
110-
// Make sure a body is returned
111-
expect(value.body).to.exist;
112109
});
113110

114111
it('should NOT update index template if desired version number equals or less than actual', async () => {

‎api-builder-plugin-fn-elasticsearch/test/unit/indices/indices-Test.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -222,7 +222,7 @@ describe('Indices rollover tests', () => {
222222

223223
it('should pass when the INDEX name is given', async () => {
224224
const mockedAliasExists = setupElasticsearchMock(client, 'indices.existsAlias', './test/unit/mock/indices/aliasExistsResponse.json', false);
225-
const mockedIndicesExists = setupElasticsearchMock(client, 'indices.exists', './test/unit/mock/dummy.json', false);
225+
const mockedIndicesExists = setupElasticsearchMock(client, 'indices.exists', './test/unit/mock/indices/indexExistsResponse.json', false);
226226

227227
const inputParameter = { index: "apigw-monitoring" };
228228
const { value, output } = await flowNode.indicesExists(inputParameter);

‎api-builder-plugin-fn-elasticsearch/test/unit/mock/indexTemplates/getTemplateUnknownTemplateResponse.json‎

Lines changed: 1 addition & 53 deletions
Original file line numberDiff line numberDiff line change
@@ -1,53 +1 @@
1-
{
2-
"body": {
3-
4-
},
5-
"statusCode": 404,
6-
"headers": {
7-
"content-type": "application/json; charset=UTF-8",
8-
"content-length": "2",
9-
"access-control-allow-credentials": "true"
10-
},
11-
"meta": {
12-
"context": null,
13-
"request": {
14-
"params": {
15-
"method": "GET",
16-
"path": "/_template/doesntexists",
17-
"body": null,
18-
"querystring": "",
19-
"headers": {
20-
"user-agent": "elasticsearch-js/7.9.1 (win32 10.0.19041-x64; Node.js v12.13.1)"
21-
},
22-
"timeout": 30000
23-
},
24-
"options": {
25-
"ignore": [
26-
404
27-
],
28-
"maxRetries": 3
29-
},
30-
"id": 1
31-
},
32-
"name": "elasticsearch-js",
33-
"connection": {
34-
"url": "http://api-env:9200/",
35-
"id": "http://api-env:9200/",
36-
"headers": {
37-
38-
},
39-
"deadCount": 0,
40-
"resurrectTimeout": 0,
41-
"_openRequests": 0,
42-
"status": "alive",
43-
"roles": {
44-
"master": true,
45-
"data": true,
46-
"ingest": true,
47-
"ml": false
48-
}
49-
},
50-
"attempts": 0,
51-
"aborted": false
52-
}
53-
}
1+
{}

‎api-builder-plugin-fn-elasticsearch/test/unit/mock/indexTemplates/putTemplateResponse.json‎

Lines changed: 2 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -1,55 +1,3 @@
11
{
2-
"body": {
3-
"acknowledged": true
4-
},
5-
"statusCode": 200,
6-
"headers": {
7-
"content-type": "application/json; charset=UTF-8",
8-
"content-length": "21",
9-
"access-control-allow-credentials": "true"
10-
},
11-
"meta": {
12-
"context": null,
13-
"request": {
14-
"params": {
15-
"method": "PUT",
16-
"path": "/_template/test-index-template",
17-
"body": "{\"order\":0,\"version\":1,\"index_patterns\":[\"apigw-traffic-summary-*\"],\"settings\":{\"index\":{\"lifecycle\":{\"name\":\"logstash-policy\",\"rollover_alias\":\"apigw-traffic-summary\"},\"codec\":\"best_compression\",\"number_of_shards\":\"5\",\"number_of_replicas\":\"1\"}},\"mappings\":{\"dynamic\":false,\"_source\":{\"enabled\":true},\"properties\":{\"processInfo.serviceId\":{\"type\":\"keyword\"},\"processInfo.version\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"http.wafStatus\":{\"type\":\"integer\"},\"serviceContext.routingPolicy\":{\"type\":\"keyword\"},\"serviceContext.faulthandlerPolicy\":{\"type\":\"keyword\"},\"processInfo.gatewayName\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"serviceContext.requestPolicy\":{\"type\":\"keyword\"},\"finalStatus\":{\"type\":\"keyword\"},\"http.uri\":{\"norms\":false,\"type\":\"text\"},\"http.localPort\":{\"type\":\"integer\"},\"serviceContext.apiVersion\":{\"type\":\"keyword\"},\"serviceContext.responsePolicy\":{\"type\":\"keyword\"},\"serviceContext.appOrg\":{\"type\":\"keyword\"},\"http.bytesReceived\":{\"type\":\"integer\"},\"serviceContext.method\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"http.localAddr\":{\"type\":\"keyword\"},\"duration\":{\"type\":\"integer\"},\"processInfo.groupName\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"protocol\":{\"type\":\"keyword\"},\"fileTransfer.serviceType\":{\"type\":\"keyword\"},\"serviceContext.apiDeprecated\":{\"type\":\"boolean\"},\"serviceContext.apiSecurity\":{\"type\":\"keyword\"},\"correlationId\":{\"type\":\"keyword\",\"doc_values\":false},\"http.method\":{\"type\":\"keyword\"},\"serviceContext.service\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"serviceContext.apiState\":{\"type\":\"keyword\"},\"serviceContext.apiOrg\":{\"type\":\"keyword\"},\"protocolSrc\":{\"norms\":false,\"type\":\"text\"},\"processInfo.groupId\":{\"type\":\"keyword\"},\"http.remoteAddr\":{\"type\":\"keyword\"},\"serviceContext.monitor\":{\"type\":\"boolean\"},\"http.remotePort\":{\"type\":\"integer\"},\"serviceContext.backendBasePath\":{\"type\":\"keyword\"},\"http.authSubjectId\":{\"type\":\"keyword\"},\"processInfo.hostname\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"processInfo.gatewayRegion\":{\"type\":\"keyword\"},\"http.vhost\":{\"type\":\"keyword\"},\"fileTransfer.direction\":{\"type\":\"keyword\"},\"@timestamp\":{\"type\":\"date\"},\"http.remoteName\":{\"type\":\"keyword\"},\"serviceContext.app\":{\"norms\":false,\"type\":\"text\",\"fields\":{\"keyword\":{\"type\":\"keyword\"}}},\"http.status\":{\"type\":\"integer\"},\"http.bytesSent\":{\"type\":\"integer\"},\"http.statusText\":{\"type\":\"keyword\"}}},\"aliases\":{}}",
18-
"querystring": "",
19-
"headers": {
20-
"user-agent": "elasticsearch-js/7.9.1 (win32 10.0.19041-x64; Node.js v12.13.1)",
21-
"content-type": "application/json",
22-
"content-length": "2515"
23-
},
24-
"timeout": 30000
25-
},
26-
"options": {
27-
"ignore": [
28-
404
29-
],
30-
"maxRetries": 3
31-
},
32-
"id": 1
33-
},
34-
"name": "elasticsearch-js",
35-
"connection": {
36-
"url": "http://api-env:9200/",
37-
"id": "http://api-env:9200/",
38-
"headers": {
39-
40-
},
41-
"deadCount": 0,
42-
"resurrectTimeout": 0,
43-
"_openRequests": 0,
44-
"status": "alive",
45-
"roles": {
46-
"master": true,
47-
"data": true,
48-
"ingest": true,
49-
"ml": false
50-
}
51-
},
52-
"attempts": 0,
53-
"aborted": false
54-
}
55-
}
2+
"acknowledged": true
3+
}

‎api-builder-plugin-fn-elasticsearch/test/unit/mock/indices/aliasExistsResponse.json‎

Lines changed: 1 addition & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,50 +1 @@
1-
{
2-
"body": true,
3-
"statusCode": 200,
4-
"headers": {
5-
"content-type": "application/json; charset=UTF-8",
6-
"content-length": "84"
7-
},
8-
"meta": {
9-
"context": null,
10-
"request": {
11-
"params": {
12-
"method": "HEAD",
13-
"path": "/_alias/apigw-monitoring",
14-
"body": null,
15-
"querystring": "",
16-
"headers": {
17-
"user-agent": "elasticsearch-js/7.9.1 (win32 10.0.19041-x64; Node.js v12.13.1)"
18-
},
19-
"timeout": 30000
20-
},
21-
"options": {
22-
"ignore": [
23-
404
24-
],
25-
"maxRetries": 3
26-
},
27-
"id": 1
28-
},
29-
"name": "elasticsearch-js",
30-
"connection": {
31-
"url": "http://api-env:9200/",
32-
"id": "http://api-env:9200/",
33-
"headers": {
34-
35-
},
36-
"deadCount": 0,
37-
"resurrectTimeout": 0,
38-
"_openRequests": 0,
39-
"status": "alive",
40-
"roles": {
41-
"master": true,
42-
"data": true,
43-
"ingest": true,
44-
"ml": false
45-
}
46-
},
47-
"attempts": 0,
48-
"aborted": false
49-
}
50-
}
1+
true

0 commit comments

Comments
 (0)