implement contract test service, not including big segments (#242)

Co-authored-by: Eli Bishop <eli@launchdarkly.com>

Author: kinyoklion

.circleci/config.yml

@@ -41,6 +41,9 @@ jobs:
         default: false
       docker-image:
         type: string
+      run-contract-tests:
+        type: boolean
+        default: true
     docker:
       - image: <<parameters.docker-image>>
     steps:
@@ -56,6 +59,14 @@ jobs:
           condition: <<parameters.run-lint>>
           steps:
             - run: npm run lint
+      - when:
+          condition: <<parameters.run-contract-tests>>
+          steps:
+            - run:
+                command: npm run contract-test-service
+                background: true
+            - run: mkdir -p reports/junit
+            - run: TEST_HARNESS_PARAMS="-junit reports/junit/contract-test-results.xml -skip-from contract-tests/testharness-suppressions.txt" npm run contract-test-harness
       - run:
           name: dependency audit
           command: ./scripts/better-audit.sh

.gitignore

@@ -1,7 +1,7 @@
 package-lock.json
 /docs/build/
 /local/
-/node_modules/
+**/node_modules/
 junit.xml
 npm-debug.log
 test-types.js

contract-tests/README.md

@@ -0,0 +1,7 @@
+# SDK contract test service
+
+This directory contains an implementation of the cross-platform SDK testing protocol defined by https://github.com/launchdarkly/sdk-test-harness. See that project's `README` for details of this protocol, and the kinds of SDK capabilities that are relevant to the contract tests. This code should not need to be updated unless the SDK has added or removed such capabilities.
+
+To run these tests locally, run `npm run contract-tests` from the SDK project root directory. This will start the test service, download the correct version of the test harness tool, and run the tests.
+
+Or, to test against an in-progress local version of the test harness, run `npm run contract-test-service` from the SDK project root directory; then, in the root directory of the `sdk-test-harness` project, build the test harness and run it from the command line.

contract-tests/index.js

@@ -0,0 +1,107 @@
+const express = require('express');
+const bodyParser = require('body-parser');
+
+const { Log } = require('./log');
+const { newSdkClientEntity, badCommandError } = require('./sdkClientEntity');
+
+const app = express();
+let server = null;
+
+const port = 8000;
+
+let clientCounter = 0;
+const clients = {};
+
+const mainLog = Log('service');
+
+app.use(bodyParser.json());
+
+app.get('/', (req, res) => {
+  res.header('Content-Type', 'application/json');
+  res.json({
+    capabilities: [
+      'server-side',
+      'all-flags-client-side-only',
+      'all-flags-details-only-for-tracked-flags',
+      'all-flags-with-reasons',
+    ],
+  });
+});
+
+app.delete('/', (req, res) => {
+  mainLog.info('Test service has told us to exit');
+  res.status(204);
+  res.send();
+
+  // Defer the following actions till after the response has been sent
+  setTimeout(() => {
+    server.close(() => process.exit());
+    // We force-quit with process.exit because, even after closing the server, there could be some
+    // scheduled tasks lingering if an SDK instance didn't get cleaned up properly, and we don't want
+    // that to prevent us from quitting.
+  }, 1);
+});
+
+app.post('/', async (req, res) => {
+  const options = req.body;
+
+  clientCounter += 1;
+  const clientId = clientCounter.toString();
+  const resourceUrl = `/clients/${clientId}`;
+
+  try {
+    const client = await newSdkClientEntity(options);
+    clients[clientId] = client;
+
+    res.status(201);
+    res.set('Location', resourceUrl);
+  } catch (e) {
+    res.status(500);
+    const message = e.message || JSON.stringify(e);
+    mainLog.error('Error creating client: ' + message);
+    res.write(message);
+  }
+  res.send();
+});
+
+app.post('/clients/:id', async (req, res) => {
+  const client = clients[req.params.id];
+  if (!client) {
+    res.status(404);
+  } else {
+    try {
+      const respValue = await client.doCommand(req.body);
+      if (respValue) {
+        res.status(200);
+        res.write(JSON.stringify(respValue));
+      } else {
+        res.status(204);
+      }
+    } catch (e) {
+      const isBadRequest = e === badCommandError;
+      res.status(isBadRequest ? 400 : 500);
+      res.write(e.message || JSON.stringify(e));
+      if (!isBadRequest && e.stack) {
+        console.log(e.stack);
+      }
+    }
+  }
+  res.send();
+});
+
+app.delete('/clients/:id', async (req, res) => {
+  const client = clients[req.params.id];
+  if (!client) {
+    res.status(404);
+    res.send();
+  } else {
+    client.close();
+    delete clients[req.params.id];
+    res.status(204);
+    res.send();
+  }
+});
+
+server = app.listen(port, () => {
+  console.log('Listening on port %d', port);
+});

contract-tests/log.js

@@ -0,0 +1,23 @@
+const ld = require('launchdarkly-node-server-sdk');
+
+function Log(tag) {
+  function doLog(level, message) {
+    console.log(new Date().toISOString() + ` [${tag}] ${level}: ${message}`);
+  }
+  return {
+    info: message => doLog('info', message),
+    error: message => doLog('error', message),
+  };
+}
+
+function sdkLogger(tag) {
+  return ld.basicLogger({
+    level: 'debug',
+    destination: line => {
+      console.log(new Date().toISOString() + ` [${tag}.sdk] ${line}`);
+    },
+  });
+}
+
+module.exports.Log = Log;
+module.exports.sdkLogger = sdkLogger;

contract-tests/package.json

@@ -0,0 +1,15 @@
+{
+  "name": "node-server-sdk-contract-tests",
+  "version": "0.0.0",
+  "main": "index.js",
+  "scripts": {
+    "start": "node index.js"
+  },
+  "author": "",
+  "license": "Apache-2.0",
+  "dependencies": {
+    "body-parser": "^1.19.0",
+    "express": "^4.17.1",
+    "launchdarkly-node-server-sdk": "file:.."
+  }
+}

contract-tests/sdkClientEntity.js

@@ -0,0 +1,109 @@
+const ld = require('launchdarkly-node-server-sdk');
+
+const { Log, sdkLogger } = require('./log');
+
+const badCommandError = new Error('unsupported command');
+
+function makeSdkConfig(options, tag) {
+  const cf = {
+    logger: sdkLogger(tag),
+  };
+  const maybeTime = seconds => (seconds === undefined || seconds === null ? undefined : seconds / 1000);
+  if (options.streaming) {
+    cf.streamUri = options.streaming.baseUri;
+    cf.streamInitialReconnectDelay = maybeTime(options.streaming.initialRetryDelayMs);
+  }
+  if (options.events) {
+    cf.allAttributesPrivate = options.events.allAttributesPrivate;
+    cf.eventsUri = options.events.baseUri;
+    cf.capacity = options.events.capacity;
+    cf.diagnosticOptOut = !options.events.enableDiagnostics;
+    cf.flushInterval = maybeTime(options.events.flushIntervalMs);
+    cf.inlineUsersInEvents = options.events.inlineUsers;
+    cf.privateAttributeNames = options.events.globalPrivateAttributes;
+  }
+  return cf;
+}
+
+async function newSdkClientEntity(options) {
+  const c = {};
+  const log = Log(options.tag);
+
+  log.info('Creating client with configuration: ' + JSON.stringify(options.configuration));
+  const timeout =
+    options.configuration.startWaitTimeMs !== null && options.configuration.startWaitTimeMs !== undefined
+      ? options.configuration.startWaitTimeMs
+      : 5000;
+  const client = ld.init(
+    options.configuration.credential || 'unknown-sdk-key',
+    makeSdkConfig(options.configuration, options.tag)
+  );
+  try {
+    await Promise.race([client.waitForInitialization(), new Promise(resolve => setTimeout(resolve, timeout))]);
+  } catch (_) {
+    // if waitForInitialization() rejects, the client failed to initialize, see next line
+  }
+  if (!client.initialized() && !options.configuration.initCanFail) {
+    client.close();
+    throw new Error('client initialization failed');
+  }
+
+  c.close = () => {
+    client.close();
+    log.info('Test ended');
+  };
+
+  c.doCommand = async params => {
+    log.info('Received command: ' + params.command);
+    switch (params.command) {
+      case 'evaluate': {
+        const pe = params.evaluate;
+        if (pe.detail) {
+          return await client.variationDetail(pe.flagKey, pe.user, pe.defaultValue);
+        } else {
+          const value = await client.variation(pe.flagKey, pe.user, pe.defaultValue);
+          return { value };
+        }
+      }
+
+      case 'evaluateAll': {
+        const pea = params.evaluateAll;
+        const eao = {
+          clientSideOnly: pea.clientSideOnly,
+          detailsOnlyForTrackedFlags: pea.detailsOnlyForTrackedFlags,
+          withReasons: pea.withReasons,
+        };
+        return { state: await client.allFlagsState(pea.user, eao) };
+      }
+
+      case 'identifyEvent':
+        client.identify(params.identifyEvent.user);
+        return undefined;
+
+      case 'customEvent': {
+        const pce = params.customEvent;
+        client.track(pce.eventKey, pce.user, pce.data, pce.metricValue);
+        return undefined;
+      }
+
+      case 'aliasEvent':
+        client.alias(params.aliasEvent.user, params.aliasEvent.previousUser);
+        return undefined;
+
+      case 'flushEvents':
+        client.flush();
+        return undefined;
+
+      case 'getBigSegmentStoreStatus':
+        return undefined;
+
+      default:
+        throw badCommandError;
+    }
+  };
+
+  return c;
+}
+
+module.exports.newSdkClientEntity = newSdkClientEntity;
+module.exports.badCommandError = badCommandError;

contract-tests/testharness-suppressions.txt

@@ -0,0 +1,2 @@
+streaming/validation/drop and reconnect if stream event has malformed JSON
+streaming/validation/drop and reconnect if stream event has well-formed JSON not matching schema

package.json

@@ -6,7 +6,10 @@
   "scripts": {
     "test": "jest --ci",
     "check-typescript": "node_modules/typescript/bin/tsc",
-    "lint": "eslint --format 'node_modules/eslint-formatter-pretty' --ignore-path .eslintignore ."
+    "lint": "eslint --format 'node_modules/eslint-formatter-pretty' --ignore-path .eslintignore .",
+    "contract-test-service": "npm --prefix contract-tests install && npm --prefix contract-tests start",
+    "contract-test-harness": "curl -s https://raw.githubusercontent.com/launchdarkly/sdk-test-harness/master/downloader/run.sh \\ | VERSION=v1 PARAMS=\"-url http://localhost:8000 -debug -stop-service-at-end $TEST_HARNESS_PARAMS\" sh",
+    "contract-tests": "npm run contract-test-service & npm run contract-test-harness"
   },
   "types": "./index.d.ts",
   "repository": {