diff --git a/rfcs/inactive_users/user_and_organization_meta_update.md b/rfcs/inactive_users/user_and_organization_meta_update.md new file mode 100644 index 000000000..e65780b70 --- /dev/null +++ b/rfcs/inactive_users/user_and_organization_meta_update.md @@ -0,0 +1,77 @@ +# User/Organization metadata update rework +## Overview and Motivation +When user or organization metadata needs to be updated, the Service uses the Redis pipeline javascript code. +For each assigned meta hash always exists a single `audience`, but there is no list of `audiences` assigned to the user or company. +To achieve easier audience tracking and a combined metadata update, I advise using a Lua based script. + +## Audience lists +Audiences stored in sets formed from `USERS_AUDIENCE` or `ORGANISATION_AUDIENCE` constants and `Id` +(eg: `{ms-users}10110110111!audiences`). Both keys contain `audience` names that are currently have assigned values. + +## utils/updateMetadata.js +Almost all logic in this file removed and ported into LUA Script. +This Function checks the consistency of the provided `opts`. If `opts.metadata` and `opts.audiences` are objects, script transforming them to an array containing these objects. Checks count of meta operations and audiences to equal each other. +Organization meta update request `utils/setOrganizationMetadata.js` uses the same functionality, so the same changes applied to it. + +After commands execution result returned from the script, decoded from JSON string. + +## script/updateMetadata.lua +Script repeats all logic including custom scripts support. + +### Script parameters: +1. KEYS[1] Audiences key template. +2. KEYS[2] used as metadata key template, eg: "{ms-users}{id}!metadata!{audience}". +3. ARGV[1] Id - organization or user-id. +4. ARGV[2] JSON encoded opts parameter opts.{script, metadata, audiences}. + +### Depending on metadata or script set: +If `opt.metadata` set: + * Script starts iterating audiences. + * On each audience, creates metadata key from provided template. + * Iterates operations from `opt.metadata`, based on index of `opts.audiences`. + ```javascript + const opts = { + audiences: ['first', 'second'], + metadata: [{ + // first audience commands + }, { + // second audience commands + }], + } + ``` + Commands execute in order: `audiences[0]` => `metadata[0]`,`audiences[1]` => `metadata[1]`, + +If `opt.script` set: +* Script iterates `audiences` and creates metadata keys from provided template + * Iterates `opt.script`: + * EVAL's script from `script.lua` and executes with params generated from: metadata keys(look to the previous step) + and passed `script.argv`. + * If script evaluation fails, script returns redis.error witch description. + +When operations/scripts processed, the script forms JSON object like +```javascript +const metaResponse = [ + //forEach audience + { + '$incr': { + field: 'result', // result returned from HINCRBY command + }, + '$remove': intCount, // count of deleted fields + '$set': "OK", // or cmd hset result. + }, +]; + +const scriptResponse = { + 'scriptName': [ + // values returned from script + ], +}; +``` + +### Audience list update +When all update operations succeeded: +* Script get's current list of user's or organization's audiences from HSET `KEYS[1]`, +unions them with `opts.audiences` and generates full list metadata keys. +* Iterates over them to check whether some data exists. +* If no data exists, the script deletes the corresponding audience from HSET `KEYS[1]`. + diff --git a/scripts/updateMetadata.lua b/scripts/updateMetadata.lua new file mode 100644 index 000000000..d3acc7e44 --- /dev/null +++ b/scripts/updateMetadata.lua @@ -0,0 +1,252 @@ +-- Script performs User/Organization metadata update and tracks used audiences +-- KEYS[1] = Audience Key template in format `{id}someExtraText` - Key stores currently used audiences associated with metadata +-- KEYS[2] = Metadata Key template in format `{id}myAvesomeMEtaKey{audience}` - Key stores metadata +-- `{id}` and `{audience}` will be replaced with real values on script runtime + +-- ARGV[1] = Id of the User/Organization which is going to be updated +-- ARGV[2] = JsonString with list of operations to execute on the metadata of the provided Id + +-- script replicates commands instead of own body +-- call of HMSET command is determined as 'non deterministic command' +-- and redis refuses to run it without this. +redis.replicate_commands() + +local audienceKeyTemplate = KEYS[1] +local metaDataTemplate = KEYS[2] +local Id = ARGV[1] +local updateOptsJson = ARGV[2] + +local scriptResult = { err = nil, ok = {}} + +-- +-- Param Validation +-- +local function isValidString(val) + if type(val) == 'string' and string.len(val) > 0 then + return true + end + return false +end + +assert(isValidString(Id), 'incorrect `id` argument') +assert(isValidString(updateOptsJson), 'incorrect `updateJson` argument') + +local updateOpts = cjson.decode(updateOptsJson) + +-- +-- Internal functions +-- + +-- evaluates provided script +local function evalLuaScript(code, environment) + if setfenv and loadstring then + local f = assert(loadstring(code)) + setfenv(f, environment) + return f + else + return assert(load(code, nil, "t", environment)) + end +end + +-- creates array with unique items from passed arrays +local function getUniqueItemsFromTables(...) + local args = {...} + local tableWithUniqueItems = {} + for _, passedTable in pairs(args) do + for __, keyName in pairs(passedTable) do + tableWithUniqueItems[keyName] = keyName + end + end + return tableWithUniqueItems +end + +-- create key from passed template, id and audience +local function makeRedisKey (template, id, audience) + local str = template:gsub('{id}', id, 1) + if audience ~= nil then + str = str:gsub('{audience}', audience, 1) + end + return str +end + +local function getResultOrSaveError(result, command, args) + if type(result) == 'table' and result['err'] ~= nil then + if (scriptResult['err'] == nil) then + scriptResult['err'] = {} + end + table.insert(scriptResult['err'], { + err = result['err'], + command = { + name = command, + args = args + } + }) + return nil + end + return result +end + +-- +-- available Meta Operations definition +-- + +-- $set: { field: value, field2: value, field3: value } +-- { HMSETResponse } +local function opSet(metaKey, args) + local setArgs = {} + + for field, value in pairs(args) do + table.insert(setArgs, field) + table.insert(setArgs, value) + end + + if #setArgs < 1 then + return nil + end + + local cmdResult = redis.pcall("HMSET", metaKey, unpack(setArgs)) + cmdResult = getResultOrSaveError(cmdResult, "HMSET", setArgs) + if cmdResult ~= nil then + return cmdResult.ok + end + + return cmdResult +end + +-- $remove: [ 'field', 'field2' ] +-- { deletedFieldsCount } - if no fields deleted or there was no such fields counter not incrementing +local function opRemove(metaKey, args) + local result = 0; + for _, field in pairs(args) do + local cmdResult = redis.pcall("HDEL", metaKey, field) + result = result + getResultOrSaveError(cmdResult, "HDEL", { metaKey, field }) + end + return result +end + +-- $incr: { field: incrValue, field2: incrValue } +-- { field: newValue } +local function opIncr(metaKey, args) + local result = {} + for field, incrVal in pairs(args) do + -- TODO fix err + local cmdResult = redis.pcall("HINCRBY", metaKey, field, incrVal) + cmdResult = getResultOrSaveError(cmdResult, "HINCRBY", { metaKey, field, incrVal }) + result[field] = cmdResult + end +-- if #result > 0 then +-- return result +-- end +-- +-- return nil + return result; +end + +-- operations index +local metaOps = { + ['$set'] = opSet, + ['$remove'] = opRemove, + ['$incr'] = opIncr +} + +-- +-- Script body +-- + +-- get list of keys to update +-- generate them from passed audiences and metaData key template +local keysToProcess = {}; +for index, audience in ipairs(updateOpts.audiences) do + local key = makeRedisKey(metaDataTemplate, Id, audience) + table.insert(keysToProcess, index, key); +end + +-- process meta update operations +if updateOpts.metaOps then + -- iterate over metadata hash field + for index, op in ipairs(updateOpts.metaOps) do + local targetOpKey = keysToProcess[index] + local metaProcessResult = {}; + + -- iterate over commands and apply them + for opName, opArg in pairs(op) do + local processFn = metaOps[opName]; + + if processFn ~= nil then + -- store command execution result + metaProcessResult[opName] = processFn(targetOpKey, opArg) + end + end + + -- store execution result of commands block + table.insert(scriptResult['ok'], metaProcessResult) + end + +-- process passed scripts +elseif updateOpts.scripts then + -- iterate over scripts and execute them in sandbox + for _, script in pairs(updateOpts.scripts) do + local env = {}; + + -- allow read access to this script scope + -- env recreated for each script to avoid scope mixing + setmetatable(env, { __index=_G }) + + -- override params to be sure that script works like it was executed like from `redis.eval` command + env.ARGV = script.argv + env.KEYS = keysToProcess + + -- evaluate script and bind to custom env + local fn = evalLuaScript(script.lua, env) + + -- run script and save result + local status, result = pcall(fn) + if status == true then + scriptResult['ok'][script.name] = result; + else + if (scriptResult['err'] == nil) then + scriptResult['err'] = {} + end + table.insert(scriptResult['err'], { + err = result, + script = script.name, + keys = keysToProcess, + args = script.args, + }) + end + + end + +end + +-- +-- Audience tracking +-- + +local audienceKey = makeRedisKey(audienceKeyTemplate, Id) +-- get saved audience list +local audiences = redis.call("SMEMBERS", audienceKey) + +-- create list containing saved and possibly new audiences +local uniqueAudiences = getUniqueItemsFromTables(audiences, updateOpts.audiences) + +-- iterate over final audience list +for _, audience in pairs(uniqueAudiences) do + -- get size of metaKey + local metaKey = makeRedisKey(metaDataTemplate, Id, audience) + local keyLen = redis.call("HLEN", metaKey) + + -- if key has data add it to the audience set + -- set members unique, so duplicates not appear + + -- if key empty or not exists (HLEN will return 0) + -- delete audience from list + if (keyLen > 0) then + redis.call("SADD", audienceKey, audience) + else + redis.call("SREM", audienceKey, audience) + end +end + +-- respond with json encoded string +return cjson.encode(scriptResult) diff --git a/src/actions/register.js b/src/actions/register.js index ba0413630..bcc2f4f38 100644 --- a/src/actions/register.js +++ b/src/actions/register.js @@ -9,7 +9,7 @@ const reduce = require('lodash/reduce'); const last = require('lodash/last'); // internal deps -const setMetadata = require('../utils/updateMetadata'); +const UserMetadata = require('../utils/metadata/user'); const redisKey = require('../utils/key'); const jwt = require('../utils/jwt'); const isDisposable = require('../utils/isDisposable'); @@ -213,7 +213,7 @@ async function performRegistration({ service, params }) { await pipeline.exec().then(handlePipeline); - await setMetadata.call(service, { + await new UserMetadata(service.redis).update({ userId, audience, metadata: audience.map((metaAudience) => ({ diff --git a/src/actions/updateMetadata.js b/src/actions/updateMetadata.js index 450762c99..47862873a 100644 --- a/src/actions/updateMetadata.js +++ b/src/actions/updateMetadata.js @@ -1,6 +1,6 @@ const omit = require('lodash/omit'); const Promise = require('bluebird'); -const updateMetadata = require('../utils/updateMetadata.js'); +const UserMetadata = require('../utils/metadata/user'); const { getUserId } = require('../utils/userData'); /** @@ -19,12 +19,15 @@ const { getUserId } = require('../utils/userData'); * @apiParam (Payload) {Object} [script] - if present will be called with passed metadata keys & username, provides direct scripting access. * Be careful with granting access to this function. */ -module.exports = function updateMetadataAction(request) { - return Promise +module.exports = async function updateMetadataAction(request) { + const userId = await Promise .bind(this, request.params.username) - .then(getUserId) - .then((userId) => ({ ...omit(request.params, 'username'), userId })) - .then(updateMetadata); + .then(getUserId); + + const userMetadata = new UserMetadata(this.redis); + const updateParams = { ...omit(request.params, 'username'), userId }; + + return userMetadata.update(updateParams); }; module.exports.transports = [require('@microfleet/core').ActionTransport.amqp]; diff --git a/src/auth/oauth/utils/attach.js b/src/auth/oauth/utils/attach.js index 1e1569c44..82974e37a 100644 --- a/src/auth/oauth/utils/attach.js +++ b/src/auth/oauth/utils/attach.js @@ -1,13 +1,13 @@ const get = require('lodash/get'); const redisKey = require('../../../utils/key'); -const updateMetadata = require('../../../utils/updateMetadata'); +const UserMetadata = require('../../../utils/metadata/user'); const handlePipeline = require('../../../utils/pipelineError'); const { USERS_SSO_TO_ID, USERS_DATA, } = require('../../../constants'); -module.exports = function attach(account, user) { +module.exports = async function attach(account, user) { const { redis, config } = this; const { id: userId } = user; const { @@ -23,17 +23,19 @@ module.exports = function attach(account, user) { // link uid to user id pipeline.hset(USERS_SSO_TO_ID, uid, userId); - return pipeline.exec().then(handlePipeline) - .bind(this) - .return({ - userId, - audience, - metadata: { - $set: { - [provider]: profile, - }, + await pipeline.exec().then(handlePipeline); + + const userMetadata = new UserMetadata(redis); + const updateParams = { + userId, + audience, + metadata: { + $set: { + [provider]: profile, }, - }) - .then(updateMetadata) - .return(profile); + }, + }; + await userMetadata.update(updateParams); + + return profile; }; diff --git a/src/auth/oauth/utils/detach.js b/src/auth/oauth/utils/detach.js index f18cfd11f..8aabafb3a 100644 --- a/src/auth/oauth/utils/detach.js +++ b/src/auth/oauth/utils/detach.js @@ -2,7 +2,7 @@ const Errors = require('common-errors'); const get = require('../../../utils/get-value'); const redisKey = require('../../../utils/key'); -const updateMetadata = require('../../../utils/updateMetadata'); +const UserMetadata = require('../../../utils/metadata/user'); const handlePipeline = require('../../../utils/pipelineError'); const { @@ -10,7 +10,7 @@ const { USERS_DATA, } = require('../../../constants'); -module.exports = function detach(provider, userData) { +module.exports = async function detach(provider, userData) { const { id: userId } = userData; const { redis, config } = this; const audience = get(config, 'jwt.defaultAudience'); @@ -28,16 +28,18 @@ module.exports = function detach(provider, userData) { // delete account reference pipeline.hdel(USERS_SSO_TO_ID, uid); - return pipeline.exec().then(handlePipeline) - .bind(this) - .return({ - userId, - audience, - metadata: { - $remove: [ - provider, - ], - }, - }) - .then(updateMetadata); + await pipeline.exec().then(handlePipeline); + + const userMetadata = new UserMetadata(redis); + const updateParams = { + userId, + audience, + metadata: { + $remove: [ + provider, + ], + }, + }; + + return userMetadata.update(updateParams); }; diff --git a/src/constants.js b/src/constants.js index 525284ecc..10f2033a2 100644 --- a/src/constants.js +++ b/src/constants.js @@ -18,6 +18,7 @@ module.exports = exports = { // hashes USERS_DATA: 'data', USERS_METADATA: 'metadata', + USERS_AUDIENCE: 'users-audiences', USERS_TOKENS: 'tokens', USERS_API_TOKENS: 'api-tokens', USERS_API_TOKENS_ZSET: 'api-tokens-set', @@ -26,6 +27,7 @@ module.exports = exports = { USERS_ORGANIZATIONS: 'user-organizations', ORGANIZATIONS_DATA: 'data', ORGANIZATIONS_METADATA: 'metadata', + ORGANIZATIONS_AUDIENCE: 'organization-audiences', ORGANIZATIONS_MEMBERS: 'members', // standard JWT with TTL diff --git a/src/custom/cappasity-users-activate.js b/src/custom/cappasity-users-activate.js index ab9826edd..42b4cdf01 100644 --- a/src/custom/cappasity-users-activate.js +++ b/src/custom/cappasity-users-activate.js @@ -1,6 +1,6 @@ const find = require('lodash/find'); const moment = require('moment'); -const setMetadata = require('../utils/updateMetadata.js'); +const UserMetadata = require('../utils/metadata/user'); /** * Adds metadata from billing into usermix @@ -13,6 +13,7 @@ module.exports = function mixPlan(userId, params) { const { payments } = config; const route = [payments.prefix, payments.routes.planGet].join('.'); const id = 'free'; + const userMetadata = new UserMetadata(this.redis); return amqp .publishAndWait(route, id, { timeout: 5000 }) @@ -20,7 +21,7 @@ module.exports = function mixPlan(userId, params) { .then(function mix(plan) { const subscription = find(plan.subs, ['name', 'month']); const nextCycle = moment().add(1, 'month').valueOf(); - const update = { + const updateParams = { userId, audience, metadata: { @@ -36,6 +37,6 @@ module.exports = function mixPlan(userId, params) { }, }; - return setMetadata.call(this, update); + return userMetadata.update(updateParams); }); }; diff --git a/src/custom/rfx-create-room-on-activate.js b/src/custom/rfx-create-room-on-activate.js index 10429ce16..469b525e2 100644 --- a/src/custom/rfx-create-room-on-activate.js +++ b/src/custom/rfx-create-room-on-activate.js @@ -1,6 +1,6 @@ const is = require('is'); const Promise = require('bluebird'); -const setMetadata = require('../utils/updateMetadata.js'); +const UserMetadata = require('../utils/metadata/user'); /** * @param {String} username @@ -22,10 +22,12 @@ function createRoom(userId, params, metadata) { name: `${metadata[audience].stationName} | ${metadata[audience].stationSchool}`, }; + const userMetadata = new UserMetadata(this.redis); + return amqp.publishAndWait(route, roomParams, { timeout: 5000 }) .bind(this) .then((room) => { - const update = { + const updateParams = { userId, audience, metadata: { @@ -35,7 +37,7 @@ function createRoom(userId, params, metadata) { }, }; - return setMetadata.call(this, update); + return userMetadata.update(updateParams); }); } diff --git a/src/utils/metadata/organization.js b/src/utils/metadata/organization.js new file mode 100644 index 000000000..0ef39d51a --- /dev/null +++ b/src/utils/metadata/organization.js @@ -0,0 +1,25 @@ +const redisKey = require('../key'); + +const MetaUpdate = require('./redis/update-metadata'); +const { ORGANIZATIONS_METADATA, ORGANIZATIONS_AUDIENCE } = require('../../constants'); + +class Organization { + constructor(redis) { + this.redis = redis; + const audienceKeyTemplate = redisKey('{id}', ORGANIZATIONS_AUDIENCE); + const metaDataTemplate = redisKey('{id}', ORGANIZATIONS_METADATA, '{audience}'); + this.metaUpdater = new MetaUpdate(this.redis, metaDataTemplate, audienceKeyTemplate); + } + + /** + * Updates metadata on a organization object + * @param {Object} opts + * @return {Promise} + */ + update(opts) { + const { organizationId, ...restOpts } = opts; + return this.metaUpdater.update({ id: organizationId, ...restOpts }); + } +} + +module.exports = Organization; diff --git a/src/utils/metadata/redis/update-metadata.js b/src/utils/metadata/redis/update-metadata.js new file mode 100644 index 000000000..6ba17a445 --- /dev/null +++ b/src/utils/metadata/redis/update-metadata.js @@ -0,0 +1,130 @@ +const { HttpStatusError } = require('common-errors'); +const { RedisError } = require('common-errors').data; +const mapValues = require('lodash/mapValues'); + +/** + * Class wraps User/Organization metadata update using atomic LUA script + */ +class UpdateMetadata { + /** + * @param redis + * @param metadataKeyTemplate + * @param audienceKeyTemplate + */ + constructor(redis, metadataKeyTemplate, audienceKeyTemplate) { + this.redis = redis; + this.audienceKeyTemplate = audienceKeyTemplate; + this.metadataKeyTemplate = metadataKeyTemplate; + } + + callLuaScript(id, ops) { + return this.redis + .updateMetadata(2, this.audienceKeyTemplate, this.metadataKeyTemplate, id, JSON.stringify(ops)); + } + + /** + * Updates metadata on a user object + * @param {Object} opts + * @return {Promise} + */ + async update(opts) { + const { + id, audience, metadata, script, + } = opts; + const audiences = Array.isArray(audience) ? audience : [audience]; + + let scriptOpts = { + audiences, + }; + + if (metadata) { + const rawMetaOps = Array.isArray(metadata) ? metadata : [metadata]; + if (rawMetaOps.length !== audiences.length) { + throw new HttpStatusError(400, 'audiences must match metadata entries'); + } + + const metaOps = rawMetaOps.map((opBlock) => UpdateMetadata.prepareOperations(opBlock)); + scriptOpts = { metaOps, ...scriptOpts }; + const updateJsonResult = await this.callLuaScript(id, scriptOpts); + return UpdateMetadata.processOpUpdateResponse(updateJsonResult); + } + + // dynamic scripts + const $scriptKeys = Object.keys(script); + const scripts = $scriptKeys.map((scriptName) => { + const { lua, argv = [] } = script[scriptName]; + return { + lua, + argv, + name: scriptName, + }; + }); + + scriptOpts = { scripts, ...scriptOpts }; + const updateResultJson = await this.callLuaScript(id, scriptOpts); + return UpdateMetadata.processLuaUpdateResponse(updateResultJson); + } + + /** + * Process results returned from LUA script when subset of Meta Operations passed + * @param jsonStr + * @returns {*} + */ + static processOpUpdateResponse(jsonStr) { + const decodedData = JSON.parse(jsonStr); + + if (decodedData.err !== undefined) { + const errors = Object.entries(decodedData.err); + const message = errors.map(([, error]) => error.err).join('; '); + + throw new RedisError(message, decodedData.err); + } + + const result = decodedData.ok.map((metaResult) => { + const opResult = {}; + for (const [key, ops] of Object.entries(metaResult)) { + if (Array.isArray(ops) && ops.length === 1) { + [opResult[key]] = ops; + } else { + opResult[key] = ops; + } + } + return opResult; + }); + + return result.length > 1 ? result : result[0]; + } + + /** + * Process results returned from LUA script when subset of LUA scripts passed + * @param jsonStr + * @returns {Response.ok|((value: any, message?: (string | Error)) => void)|string|boolean} + */ + static processLuaUpdateResponse(jsonStr) { + const decodedData = JSON.parse(jsonStr); + + if (decodedData.err !== undefined) { + const errors = Object.entries(decodedData.err); + const message = errors.map(([, error]) => `Script: ${error.script} Failed with error: ${error.err}`).join('; '); + throw new RedisError(message, decodedData.err); + } + + return decodedData.ok; + } + + /** + * Encodes operation field values ito json string + * If encoding performed in LUA script using CJSON lib, empty arrays become empty objects. + * This breaks logic + * @param metaOps + * @returns {*} + */ + static prepareOperations(ops) { + if (Object.hasOwnProperty.call(ops, '$set')) { + ops.$set = mapValues(ops.$set, JSON.stringify); + } + return ops; + } +} + +module.exports = UpdateMetadata; diff --git a/src/utils/metadata/user.js b/src/utils/metadata/user.js new file mode 100644 index 000000000..a0e3cc3ea --- /dev/null +++ b/src/utils/metadata/user.js @@ -0,0 +1,25 @@ +const redisKey = require('../key'); + +const MetaUpdate = require('./redis/update-metadata'); +const { USERS_METADATA, USERS_AUDIENCE } = require('../../constants'); + +class User { + constructor(redis) { + this.redis = redis; + const audienceKeyTemplate = redisKey('{id}', USERS_AUDIENCE); + const metaDataTemplate = redisKey('{id}', USERS_METADATA, '{audience}'); + this.metaUpdater = new MetaUpdate(this.redis, metaDataTemplate, audienceKeyTemplate); + } + + /** + * Updates metadata on a user object + * @param {Object} opts + * @return {Promise} + */ + update(opts) { + const { userId, ...restOpts } = opts; + return this.metaUpdater.update({ id: userId, ...restOpts }); + } +} + +module.exports = User; diff --git a/src/utils/organization/registerOrganizationMembers.js b/src/utils/organization/registerOrganizationMembers.js index a97c1fa2a..d5d8161a0 100644 --- a/src/utils/organization/registerOrganizationMembers.js +++ b/src/utils/organization/registerOrganizationMembers.js @@ -14,7 +14,7 @@ const { USERS_ID_FIELD, } = require('../../constants.js'); const scrypt = require('../scrypt'); -const setMetadata = require('../updateMetadata'); +const UserMetadata = require('../metadata/user'); async function registerOrganizationMember(member) { const { redis, config } = this; @@ -36,7 +36,8 @@ async function registerOrganizationMember(member) { pipeline.hset(USERS_USERNAME_TO_ID, email, userId); await pipeline.exec().then(handlePipeline); - await setMetadata.call(this, { + const userMetadata = new UserMetadata(redis); + await userMetadata.update({ userId, audience, metadata: [{ diff --git a/src/utils/setOrganizationMetadata.js b/src/utils/setOrganizationMetadata.js index a9f9b47db..2a2f8eba1 100644 --- a/src/utils/setOrganizationMetadata.js +++ b/src/utils/setOrganizationMetadata.js @@ -1,11 +1,5 @@ /* eslint-disable no-mixed-operators */ -const Promise = require('bluebird'); -const is = require('is'); -const { HttpStatusError } = require('common-errors'); -const redisKey = require('../utils/key.js'); -const handlePipeline = require('../utils/pipelineError.js'); -const { handleAudience } = require('../utils/updateMetadata.js'); -const { ORGANIZATIONS_METADATA } = require('../constants.js'); +const OrganizationMetadata = require('../utils/metadata/organization'); /** * Updates metadata on a organization object @@ -13,29 +7,7 @@ const { ORGANIZATIONS_METADATA } = require('../constants.js'); * @return {Promise} */ async function setOrganizationMetadata(opts) { - const { redis } = this; - const { - organizationId, audience, metadata, - } = opts; - const audiences = is.array(audience) ? audience : [audience]; - - // keys - const keys = audiences.map((aud) => redisKey(organizationId, ORGANIZATIONS_METADATA, aud)); - - // if we have meta, then we can - if (metadata) { - const pipe = redis.pipeline(); - const metaOps = is.array(metadata) ? metadata : [metadata]; - - if (metaOps.length !== audiences.length) { - return Promise.reject(new HttpStatusError(400, 'audiences must match metadata entries')); - } - - metaOps.forEach((meta, idx) => handleAudience(pipe, keys[idx], meta)); - return pipe.exec().then(handlePipeline); - } - - return true; + return new OrganizationMetadata(this.redis).update(opts); } module.exports = setOrganizationMetadata; diff --git a/src/utils/updateMetadata.js b/src/utils/updateMetadata.js deleted file mode 100644 index 6fa5f56b8..000000000 --- a/src/utils/updateMetadata.js +++ /dev/null @@ -1,144 +0,0 @@ -/* eslint-disable no-mixed-operators */ -const Promise = require('bluebird'); -const mapValues = require('lodash/mapValues'); -const is = require('is'); -const { HttpStatusError } = require('common-errors'); -const redisKey = require('../utils/key.js'); -const sha256 = require('./sha256.js'); -const handlePipeline = require('../utils/pipelineError.js'); -const { USERS_METADATA } = require('../constants.js'); - -const JSONStringify = (data) => JSON.stringify(data); - -/** - * Process metadata update operation for a passed audience - * @param {Object} pipeline - * @param {String} audience - * @param {Object} metadata - */ -function handleAudience(pipeline, key, metadata) { - const { $remove } = metadata; - const $removeOps = $remove && $remove.length || 0; - if ($removeOps > 0) { - pipeline.hdel(key, $remove); - } - - const { $set } = metadata; - const $setKeys = $set && Object.keys($set); - const $setLength = $setKeys && $setKeys.length || 0; - if ($setLength > 0) { - pipeline.hmset(key, mapValues($set, JSONStringify)); - } - - const { $incr } = metadata; - const $incrFields = $incr && Object.keys($incr); - const $incrLength = $incrFields && $incrFields.length || 0; - if ($incrLength > 0) { - $incrFields.forEach((fieldName) => { - pipeline.hincrby(key, fieldName, $incr[fieldName]); - }); - } - - return { - $removeOps, $setLength, $incrLength, $incrFields, - }; -} - -/** - * Maps updateMetadata ops - * @param {Array} responses - * @param {Array} operations - * @return {Object|Array} - */ -function mapMetaResponse(operations, responses) { - let cursor = 0; - return Promise - .map(operations, (props) => { - const { - $removeOps, $setLength, $incrLength, $incrFields, - } = props; - const output = {}; - - if ($removeOps > 0) { - output.$remove = responses[cursor]; - cursor += 1; - } - - if ($setLength > 0) { - output.$set = responses[cursor]; - cursor += 1; - } - - if ($incrLength > 0) { - const $incrResponse = output.$incr = {}; - $incrFields.forEach((fieldName) => { - $incrResponse[fieldName] = responses[cursor]; - cursor += 1; - }); - } - - return output; - }) - .then((ops) => (ops.length > 1 ? ops : ops[0])); -} - -/** - * Handle script, mutually exclusive with metadata - * @param {Array} scriptKeys - * @param {Array} responses - */ -function mapScriptResponse(scriptKeys, responses) { - const output = {}; - scriptKeys.forEach((fieldName, idx) => { - output[fieldName] = responses[idx]; - }); - return output; -} - -/** - * Updates metadata on a user object - * @param {Object} opts - * @return {Promise} - */ -function updateMetadata(opts) { - const { redis } = this; - const { - userId, audience, metadata, script, - } = opts; - const audiences = is.array(audience) ? audience : [audience]; - - // keys - const keys = audiences.map((aud) => redisKey(userId, USERS_METADATA, aud)); - - // if we have meta, then we can - if (metadata) { - const pipe = redis.pipeline(); - const metaOps = is.array(metadata) ? metadata : [metadata]; - - if (metaOps.length !== audiences.length) { - return Promise.reject(new HttpStatusError(400, 'audiences must match metadata entries')); - } - - const operations = metaOps.map((meta, idx) => handleAudience(pipe, keys[idx], meta)); - return pipe.exec() - .then(handlePipeline) - .then((res) => mapMetaResponse(operations, res)); - } - - // dynamic scripts - const $scriptKeys = Object.keys(script); - const scripts = $scriptKeys.map((scriptName) => { - const { lua, argv = [] } = script[scriptName]; - const sha = sha256(lua); - const name = `ms_users_${sha}`; - if (!is.fn(redis[name])) { - redis.defineCommand(name, { lua }); - } - return redis[name](keys.length, keys, argv); - }); - - return Promise.all(scripts).then((res) => mapScriptResponse($scriptKeys, res)); -} - -updateMetadata.handleAudience = handleAudience; -module.exports = updateMetadata; diff --git a/test/suites/updateMetadata.js b/test/suites/updateMetadata.js index ef0a67265..3ec6733df 100644 --- a/test/suites/updateMetadata.js +++ b/test/suites/updateMetadata.js @@ -36,6 +36,7 @@ describe('#updateMetadata', function getMetadataSuite() { .reflect() .then(inspectPromise()) .then((data) => { + console.log(data); expect(data.$remove).to.be.eq(0); }); }); @@ -64,6 +65,7 @@ describe('#updateMetadata', function getMetadataSuite() { $incr: { b: 2, }, + $remove: ['c'], }, { $incr: { @@ -75,15 +77,14 @@ describe('#updateMetadata', function getMetadataSuite() { .then(inspectPromise()) .then((data) => { const [mainData, extraData] = data; - + console.log(data); expect(mainData.$set).to.be.eq('OK'); expect(mainData.$incr.b).to.be.eq(2); expect(extraData.$incr.b).to.be.eq(3); }); }); - it('must be able to run dynamic scripts', function test() { - const dispatch = simpleDispatcher(this.users.router); + it('must be able to run dynamic scripts', async function test() { const params = { username, audience: [audience, extra], @@ -95,15 +96,39 @@ describe('#updateMetadata', function getMetadataSuite() { }, }; - return dispatch('users.updateMetadata', params) - .reflect() - .then(inspectPromise()) - .then((data) => { - expect(data.balance).to.be.deep.eq([ - `{ms-users}${this.userId}!metadata!${audience}`, - `{ms-users}${this.userId}!metadata!${extra}`, - 'nom-nom', - ]); - }); + const updated = await this.dispatch('users.updateMetadata', params); + + expect(updated.balance).to.be.deep.eq([ + `{ms-users}${this.userId}!metadata!${audience}`, + `{ms-users}${this.userId}!metadata!${extra}`, + 'nom-nom', + ]); + }); + + it('must be able to run dynamic scripts / default namespace available', async function test() { + const lua = ` + local t = {} + table.insert(t, "foo") + local jsonDec = cjson.decode('{"bar": 1}') + local typeCheck = type(t) + redis.call("SET", "fookey", 777); + return {jsonDec.bar, redis.call("TIME"), redis.call("GET", "fookey"), typeCheck, unpack(t)} + `; + + const params = { + username, + audience: [audience], + script: { + check: { + lua, + argv: ['nom-nom'], + }, + }, + }; + const updated = await this.dispatch('users.updateMetadata', params); + const [jsonVal, redisTime, keyValue] = updated.check; + expect(jsonVal).to.be.eq(1); + expect(redisTime).to.be.an('array'); + expect(keyValue).to.be.eq('777'); }); }); diff --git a/test/suites/utils/metadata/redis/update-metadata.js b/test/suites/utils/metadata/redis/update-metadata.js new file mode 100644 index 000000000..bf21c85bd --- /dev/null +++ b/test/suites/utils/metadata/redis/update-metadata.js @@ -0,0 +1,152 @@ +const { expect } = require('chai'); +const { RedisError } = require('common-errors').data; + +const UpdateMetaData = require('../../../../../src/utils/metadata/redis/update-metadata'); +// direct access test suite. Validator doesn't allow us to use incorrect arguments +describe('#updateMetadata LUA script', function updateMetadataLuaSuite() { + const id = '7777777777777'; + const audience = '*.localhost'; + let metaUpdater; + + before(global.startService.bind(this)); + afterEach(global.clearRedis.bind(this, true)); + after(global.clearRedis.bind(this)); + + beforeEach('setUserProps', async () => { + const params = { + id, + audience: [ + audience, + '*.extra', + ], + metadata: [ + { + $set: { + x: 10, + b: 12, + c: 'cval', + }, + }, { + $set: { + x: 20, + b: 22, + c: 'xval', + }, + }, + ], + }; + + metaUpdater = new UpdateMetaData(this.users.redis, '{id}:testMeta:{audience}', '{id}:audience'); + await metaUpdater.update(params); + }); + + it('sets meta', async () => { + const redisUserMetaKey = `${id}:testMeta:${audience}`; + const userDataAudience = await this.users.redis.hgetall(redisUserMetaKey); + expect(userDataAudience).to.include({ x: '10', c: '"cval"', b: '12' }); + + const userDataExtraAudience = await this.users.redis.hgetall(`${id}:testMeta:*.extra`); + expect(userDataExtraAudience).to.include({ x: '20', c: '"xval"', b: '22' }); + }); + + it('tracks audienceList', async () => { + const audiencesList = await this.users.redis.smembers(`${id}:audience`); + expect(audiencesList).to.include.members(['*.localhost', '*.extra']); + }); + + it('tracks audienceList after remove', async () => { + await metaUpdater.update({ + id, + audience: [ + '*.extra', + ], + metadata: [ + { + $remove: ['x', 'c', 'b'], + }, + ], + }); + + const audiencesList = await this.users.redis.smembers(`${id}:audience`); + expect(audiencesList).to.be.deep.equal(['*.localhost']); + }); + + // should error if one of the commands failed to run + // BUT other commands must be executed + it('behaves like Redis pipeline using MetaOperations', async () => { + const params = { + id, + audience: [ + audience, + ], + metadata: [ + { + $set: { + x: 10, + y: null, + }, + $incr: { + b: 2, + d: 'asf', + }, + $remove: ['c'], + }, + ], + }; + + let updateError; + try { + await metaUpdater.update(params); + } catch (e) { + updateError = e; + } + expect(updateError).to.be.an.instanceof(RedisError, 'should throw error'); + + const redisUserMetaKey = `${id}:testMeta:${audience}`; + const userData = await this.users.redis.hgetall(redisUserMetaKey); + + expect(userData).to.include({ x: '10', b: '14' }); + expect(userData).to.not.include({ c: 'cval' }); + }); + + it('executes LUA scripts despite on some of the scripts error', async () => { + const luaScript = ` + redis.call("SET", '{ms-users}myTestKey' .. ARGV[1], ARGV[1]) + return ARGV[1] + `; + + const params = { + id, + audience: [audience], + script: { + firstScript: { + lua: 'return foo', + }, + secondScript: { + lua: luaScript, + argv: ['777'], + }, + thirdScript: { + lua: luaScript, + argv: ['888'], + }, + }, + }; + + let updateError; + + try { + await metaUpdater.update(params); + } catch (e) { + updateError = e; + } + + expect(updateError).to.be.an.instanceof(RedisError, 'should throw error'); + + const testKeyContents = await this.users.redis.get('myTestKey777'); + expect(testKeyContents).to.be.equal('777'); + + const secondTestKeyContents = await this.users.redis.get('myTestKey888'); + expect(secondTestKeyContents).to.be.equal('888'); + }); +});