From c3388ce48040992ab1883ed49cb05722cb8522fe Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Tue, 1 Nov 2022 11:04:50 -0500 Subject: [PATCH 01/24] Add action to retrieve region code by name --- services/locale/src/actions/region/get-code.js | 14 ++++++++++++++ services/locale/src/actions/region/index.js | 2 ++ 2 files changed, 16 insertions(+) create mode 100644 services/locale/src/actions/region/get-code.js diff --git a/services/locale/src/actions/region/get-code.js b/services/locale/src/actions/region/get-code.js new file mode 100644 index 00000000..e294b75f --- /dev/null +++ b/services/locale/src/actions/region/get-code.js @@ -0,0 +1,14 @@ +const { createRequiredParamError } = require('@base-cms/micro').service; +const regions = require('../../regions'); + +module.exports = ({ name, countryCode }) => { + if (!countryCode) throw createRequiredParamError('countryCode'); + if (!name) throw createRequiredParamError('name'); + const options = regions[countryCode] || {}; + const found = Object.entries(options).find(([, obj]) => obj.name === name); + if (found) { + const [, { regionCode }] = found; + return regionCode; + } + return false; +}; diff --git a/services/locale/src/actions/region/index.js b/services/locale/src/actions/region/index.js index 12433613..520de6d5 100644 --- a/services/locale/src/actions/region/index.js +++ b/services/locale/src/actions/region/index.js @@ -1,6 +1,7 @@ const asObject = require('./as-object'); const getAll = require('./get-all'); const getAllFor = require('./get-all-for'); +const getCode = require('./get-code'); const getName = require('./get-name'); const isValid = require('./is-valid'); @@ -8,6 +9,7 @@ module.exports = { asObject, getAll, getAllFor, + getCode, getName, isValid, }; From 59736e9d2f49b67b2ec28304eb69909dc040e2e8 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Tue, 1 Nov 2022 11:07:01 -0500 Subject: [PATCH 02/24] Create SMG import service --- .gitignore | 2 + docker-compose.yml | 14 ++ services/import/.eslintrc.js | 9 ++ services/import/data/.gitignore | 0 services/import/gulpfile.js | 9 ++ services/import/package.json | 33 +++++ services/import/src/index.js | 87 +++++++++++ services/import/src/parse-csv.js | 11 ++ services/import/src/upsert.js | 5 + services/import/src/validate.js | 209 +++++++++++++++++++++++++++ yarn.lock | 241 ++++++++++++++++++++++++++++++- 11 files changed, 619 insertions(+), 1 deletion(-) create mode 100644 services/import/.eslintrc.js create mode 100644 services/import/data/.gitignore create mode 100644 services/import/gulpfile.js create mode 100644 services/import/package.json create mode 100644 services/import/src/index.js create mode 100644 services/import/src/parse-csv.js create mode 100644 services/import/src/upsert.js create mode 100644 services/import/src/validate.js diff --git a/.gitignore b/.gitignore index 0daa0a27..9b700d62 100644 --- a/.gitignore +++ b/.gitignore @@ -87,3 +87,5 @@ typings/ # DynamoDB Local files .dynamodb/ .DS_Store + +services/import/data/**/*.csv diff --git a/docker-compose.yml b/docker-compose.yml index 62979563..a13cb20f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -206,6 +206,20 @@ services: ports: - "12011:80" + import: + <<: *node + working_dir: /identity-x/services/import + command: ["node", "."] + environment: + <<: *env + INTERNAL_PORT: 80 + EXTERNAL_PORT: 12012 + depends_on: + - application + - locale + ports: + - "12012:80" + volumes: mongodb: {} yarn-cache: {} diff --git a/services/import/.eslintrc.js b/services/import/.eslintrc.js new file mode 100644 index 00000000..b73a5afa --- /dev/null +++ b/services/import/.eslintrc.js @@ -0,0 +1,9 @@ +module.exports = { + extends: 'airbnb-base', + plugins: [ + 'import' + ], + rules: { + 'no-underscore-dangle': [ 'error', { allow: ['_id'] } ], + }, +}; diff --git a/services/import/data/.gitignore b/services/import/data/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/services/import/gulpfile.js b/services/import/gulpfile.js new file mode 100644 index 00000000..09eeb244 --- /dev/null +++ b/services/import/gulpfile.js @@ -0,0 +1,9 @@ +const gulpfile = require('../../gulpfile'); + +gulpfile({ + entry: 'src/index.js', + lintPaths: ['src/**/*.js'], + watchPaths: [ + 'src/**/*.js', + ], +}); diff --git a/services/import/package.json b/services/import/package.json new file mode 100644 index 00000000..c5b033ce --- /dev/null +++ b/services/import/package.json @@ -0,0 +1,33 @@ +{ + "name": "@identity-x/import-service", + "version": "1.11.2", + "description": "The IdentityX import service.", + "main": "src/index.js", + "author": "Josh Worden ", + "repository": "https://github.com/base-cms/id-me/tree/master/services/import", + "license": "MIT", + "private": true, + "scripts": { + "dev": "gulp", + "lint": "eslint --ext .js --max-warnings 5 ./", + "test": "yarn lint" + }, + "dependencies": { + "@base-cms/env": "^1.0.0", + "@base-cms/object-path": "^1.9.0", + "@identity-x/service-clients": "^1.2.0", + "@identity-x/utils": "^1.2.0", + "async": "^3.2.4", + "csv2json": "^2.0.2", + "inquirer": "^7.3.3", + "validator": "^11.0.0" + }, + "devDependencies": { + "eslint": "^5.16.0", + "eslint-config-airbnb-base": "^13.1.0", + "eslint-plugin-import": "^2.17.2", + "gulp": "^4.0.2", + "gulp-cached": "^1.1.1", + "gulp-eslint": "^5.0.0" + } +} diff --git a/services/import/src/index.js b/services/import/src/index.js new file mode 100644 index 00000000..5f4040b1 --- /dev/null +++ b/services/import/src/index.js @@ -0,0 +1,87 @@ +const fs = require('fs'); +const { join } = require('path'); +const inquirer = require('inquirer'); +const { applicationService } = require('@identity-x/service-clients'); +const parseCSV = require('./parse-csv'); +const validate = require('./validate'); +const upsert = require('./upsert'); + +const { log } = console; + +process.on('unhandledRejection', (e) => { + log(e); + throw e; +}); + +const findFilesIn = (path, ext = 'csv', arr = []) => { + const pattern = new RegExp(`.${ext}$`, 'i'); + let found = arr || []; + fs.readdirSync(path).forEach((file) => { + const filePath = `${path}/${file}`; + if (fs.statSync(filePath).isDirectory()) { + found = findFilesIn(filePath, ext, found); + } else if (pattern.test(file)) { + found.push(join(filePath)); + } + }); + return found; +}; + +(async () => { + const { + appId, + file, + limit, + errorOnBadAnswer, + } = await inquirer.prompt([ + { + type: 'input', + name: 'orgId', + message: 'What organization should be imported into?', + default: '627aa459dfa0e102fdc93122', // SMG + }, + { + type: 'list', + name: 'appId', + message: 'What application should be imported into?', + choices: async (ans) => { + const apps = await applicationService.request('listForOrg', { id: ans.orgId, fields: { name: 1 } }); + return apps.map(app => ({ name: app.name, value: app._id })); + }, + default: '629bac8439347cfce3861789', // Lab Pulse + }, + { + type: 'list', + name: 'file', + message: 'Which file should be imported?', + choices: () => { + const path = join(__dirname, '../data'); + return findFilesIn(path); + }, + }, + { + type: 'number', + name: 'limit', + message: 'How many users should be created/validated at once?', + default: 100, + }, + { + type: 'boolean', + name: 'errorOnBadAnswer', + message: 'Should record be skipped if a bad answer value is found?', + default: true, + }, + ]); + + + try { + log(`Importing records from ${file} to ${appId}!`); + const records = await parseCSV(file); + const validated = await validate(records, appId, limit, errorOnBadAnswer); + await upsert(validated, appId); + log('Import complete!'); + } catch (e) { + log('Encountered error!', e); + process.exit(1); + } +})().catch((e) => { throw e; }); diff --git a/services/import/src/parse-csv.js b/services/import/src/parse-csv.js new file mode 100644 index 00000000..759deb79 --- /dev/null +++ b/services/import/src/parse-csv.js @@ -0,0 +1,11 @@ +const csv2json = require('csv2json'); +const fs = require('fs'); + +module.exports = filename => new Promise(async (resolve, reject) => { + const strings = []; + fs.createReadStream(filename) + .pipe(csv2json({})) + .on('data', e => strings.push(e.toString())) + .on('error', reject) + .on('end', () => resolve(JSON.parse(strings.join('')))); +}); diff --git a/services/import/src/upsert.js b/services/import/src/upsert.js new file mode 100644 index 00000000..0d761a4e --- /dev/null +++ b/services/import/src/upsert.js @@ -0,0 +1,5 @@ +const { log } = console; + +module.exports = async (records = [], applicationId, limit = 10) => { + log('Upserting ', records.length, applicationId, limit); +}; diff --git a/services/import/src/validate.js b/services/import/src/validate.js new file mode 100644 index 00000000..0d2d9ff4 --- /dev/null +++ b/services/import/src/validate.js @@ -0,0 +1,209 @@ +const validator = require('validator'); +const { applicationService, localeService } = require('@identity-x/service-clients'); +const { normalizeEmail } = require('@identity-x/utils'); +// const { eachLimit } = require('async'); +const eachLimit = require('async/eachLimit'); + +const { log } = console; +const regionMap = new Map(); +const countryMap = new Map(); +const fieldMap = new Map(); +const answerMap = new Map(); +const oldAnswerMap = new Map([ + // Field: Technologies + ['Analyzers & Reagents', ['Molecular Diagnostics', 'Immunoassay', 'Clinical Chemistry']], + ['Chromatography', ['Emerging Technologies']], + ['Digital Pathology', ['Pathology and Histology']], + ['Histology', ['Pathology and Histology']], + ['Lab Automation Software', ['Lab Software']], + ['Lab Automation', ['Automation']], + ['Laboratory Information Systems (LIS)', ['Lab Software']], + ['Microscopy & Imaging', ['Pathology and Histology']], + ['Molecular Diagnostics', ['Molecular Diagnostics', 'Sequencing', 'Liquid Biopsy', 'Genomics']], // re-route unchanged @todo + ['Sequencing', ['Molecular Diagnostics', 'Sequencing', 'Liquid Biopsy', 'Genomics']], // re-route unchanged @todo + + // Field: Specialities + ['Autoimmune Testing', ['Autoimmune']], + ['Cancer Diagnostics', ['Cancer']], + ['Clinical Pathology', []], // remove + ['CNS Testing', []], // remove + ['Companion Diagnostics', []], // remove + ['Disease Patterns and Trends', []], // remove + ['Economics & Lab Management', []], // remove + ['Emergency Medicine', ['Emergency medicine']], + ['Genomics', []], // remove + ['Microbiology/Infectious Diseases', ['Infectious disease/microbiology']], + ['New Viral Threats', ['COVID-19/new disease threats']], + ['Precision Medicine', []], // remove + ['Routine Testing', ['Routine testing']], + ['Sexually transmitted diseases', []], // remove + ['Substance Abuse Testing', ['Drugs of Abuse/Toxicology']], + ['Urgent Care', ['Urgent care']], + + // Field: Profession + ['Histotechnologist', ['histotechnologist']], + ['Lab Technician', ['Lab technician']], + ['Pathologist', ['pathologist']], + + // Field: Org Type + ['Academic Institution', ['Academic institution']], + ['CLIA lab', ['CLIA laboratory']], + ['Consulting firm', ['Contract research organization']], + ['Group purchasing org', []], // remove + ['Other', []], // remove + ['Pharmaceutical', ['Diagnostic company/test developer/manufacturer']], + ['Staffing/Service', ['Diagnostic company/test developer/manufacturer']], + ['Staffing/Services', ['Diagnostic company/test developer/manufacturer']], + ['Vendor', ['Diagnostic company/test developer/manufacturer']], +]); +const badAnswers = []; + +const getRegionCode = async (name, countryCode) => { + if (regionMap.has(name)) return regionMap.get(name); + + const code = await localeService.request('region.getCode', { name, countryCode }); + // log('getRegionCode', { name, countryCode, code }); + if (code) { + regionMap.set(name, code); + return code; + } + // log('resp', code); + throw new Error(`unknown region name "${name}"`); +}; + +const getCountryCode = async (name) => { + if (countryMap.has(name)) return countryMap.get(name); + const code = await localeService.request('country.getCode', { name }); + // log('getCountryCode', { name, code }); + if (code) { + countryMap.set(name, code); + return code; + } + throw new Error(`Unknown country name: "${name}"`); +}; + +const mapBooleanAnswers = async (data) => { + const keys = Object.keys(data) + .filter(key => /^Custom:/i.test(key)) + .filter(key => fieldMap.get(key).type === 'boolean'); + return keys.map((key) => { + const k = fieldMap.get(key); + const value = data[key] === 'TRUE'; + return { _id: k.id, value }; + }).filter(v => v); +}; + +const mapSelectAnswers = async (data, error = false) => { + const keys = Object.keys(data) + .filter(key => /^Custom:/i.test(key)) + .filter(key => data[key]) + .filter(key => fieldMap.get(key).type === 'select'); + return keys.map((key) => { + const k = fieldMap.get(key); + const answers = `${data[key]}`.split('|'); + const values = answers.reduce((arr, v) => { + const value = `${v}`.trim(); + if (oldAnswerMap.has(value)) { + return [...arr, ...oldAnswerMap.get(value).map(a => answerMap.get(a))]; + } + if (!answerMap.has(value)) { + badAnswers[key] = badAnswers[key] || new Set(); + badAnswers[key].add(value); + if (error) throw new Error(`Unable to find mapped answer for "${value}"!`); + return arr; + } + return [...arr, answerMap.get(value)]; + }, []); + // if (values.length !== answers.length) { + // log('v', values, answers); + // process.exit(1); + // } + return { _id: k.id, answers, values }; + }).filter(v => v); +}; + +module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswer = false) => { + const valid = []; + const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 } }); + answers.edges.forEach(({ node }) => { + // eslint-disable-next-line no-underscore-dangle + fieldMap.set(`Custom: ${node.name}`, { id: node._id, type: node._type }); + if (node.options) { + node.options.forEach((option) => { + answerMap.set(option.label, option._id); + }); + } + }); + + log('Validating', records.length, applicationId); + + await eachLimit(records, limit, async (record) => { + const isInternal = /[a-f0-9]{24,}/i.test(record._id); + try { + const filtered = Object.keys(record).reduce((obj, key) => ({ + ...obj, + ...(record[key] && /^Custom:/.test(key) === false && { [key]: record[key] }), + // Fix bad data + ...(record.countryName === 'United States' && { countryName: 'United States of America' }), + ...(record.countryName === 'Russia' && { countryName: 'Russian Federation' }), + ...(record.countryName === 'Iran' && { countryName: 'Iran, Islamic Republic of' }), + ...(record.countryName === 'Vietnam' && { countryName: 'Viet Nam' }), + ...(record.countryName === 'Korea, Republic of' && { countryName: 'South Korea' }), + ...(record.countryName === 'Macedonia' && { countryName: 'North Macedonia, Republic of' }), + ...(record.countryName === 'Rwandese Republic' && { countryName: 'Rwanda' }), + // Regions + ...(record.regionName === 'Toscana' && { countryName: 'Italy' }), + ...(record.regionName === 'Mexico City' && { regionName: undefined }), + ...(record.regionName === 'DC' && { regionName: 'District of Columbia' }), + // Canada, eh + ...(record.countryName === 'Canada' && { + ...(record.regionName === 'AB' && { regionName: 'Alberta' }), + ...(record.regionName === 'BC' && { regionName: 'British Columbia' }), + ...(record.regionName === 'manitoba' && { regionName: 'Manitoba' }), + ...(record.regionName === 'NS' && { regionName: 'Nova Scotia' }), + ...(record.regionName === 'Qc' && { regionName: 'Quebec' }), + ...(record.regionName === 'Newfoundland' && { regionName: 'Newfoundland and Labrador' }), + ...(record.regionName === 'ONTARIO' && { regionName: 'Ontario' }), + }), + + // Bad data + ...(['AF5B5252903A4', 'CD79C0974D719', 'Choose One'].includes(record.countryName) && { + countryName: undefined, + }), + ...(['AF5B5252903A4', 'CD79C0974D719', 'Choose One'].includes(record.regionName) && { + regionName: undefined, + }), + }), {}); + const { countryName } = filtered; + const countryCode = countryName ? await getCountryCode(countryName) : undefined; + const validCC = ['US', 'MX', 'CA'].includes(countryCode); + const normalized = { + ...filtered, + ...(!isInternal && { _id: undefined, externalId: `backoffice.smg.member*${Buffer.from(filtered._id).toString('base64')}` }), + email: normalizeEmail(filtered.email), + // domain: + verified: false, + receiveEmail: filtered.receiveEmail === 'TRUE', + // country is valid/not typo'd + ...(countryName && { countryCode }), + // region/state codes are set properly + ...(filtered.regionName && validCC && { + regionCode: await getRegionCode(filtered.regionName, countryCode), + }), + // values for custom questions map to valid answers + customBooleanFieldAnswers: await mapBooleanAnswers(record), + customSelectFieldAnswers: await mapSelectAnswers(record, errorOnBadAnswer), + }; + if (!validator.isEmail(normalized.email)) throw new Error(`${normalized.email} is not a valid email address.`); + // log(normalized); + valid.push(normalized); + } catch (e) { + log('record failed', e); + // throw e; + } + }); + + log(badAnswers); + + return valid; +}; diff --git a/yarn.lock b/yarn.lock index 2a3da7a5..df0cf788 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3150,6 +3150,13 @@ ansi-escapes@^3.0.0, ansi-escapes@^3.2.0: resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-3.2.0.tgz#8780b98ff9dbf5638152d1f1fe5c1d7b4442976b" integrity sha512-cBhpre4ma+U0T1oM5fXg7Dy1Jw7zzwv7lt/GoCpr+hDQJoYnKVPLL4dCvSEFMmQurOQvSrwT7SL/DAlhBI97RQ== +ansi-escapes@^4.2.1: + version "4.3.2" + resolved "https://registry.yarnpkg.com/ansi-escapes/-/ansi-escapes-4.3.2.tgz#6b2291d1db7d98b6521d5f1efa42d0f3a9feb65e" + integrity sha512-gKXj5ALrKWQLsYG9jlTRmR/xKluxHV+Z9QEwNIgCfM1/uwPMCuzVVnh5mwTd+OuBZcwSIMbqssNWRm1lE51QaQ== + dependencies: + type-fest "^0.21.3" + ansi-gray@^0.1.1: version "0.1.1" resolved "https://registry.yarnpkg.com/ansi-gray/-/ansi-gray-0.1.1.tgz#2962cf54ec9792c48510a3deb524436861ef7251" @@ -3172,6 +3179,11 @@ ansi-regex@^4.1.0: resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-4.1.0.tgz#8b9f8f08cf1acb843756a839ca8c7e3168c51997" integrity sha512-1apePfXM1UOSqw0o9IiFAovVz9M5S1Dg+4TrDwfMewQ6p/rmMueb7tWZjQ1rx4Loy1ArBggoqGpfqqdI4rondg== +ansi-regex@^5.0.1: + version "5.0.1" + resolved "https://registry.yarnpkg.com/ansi-regex/-/ansi-regex-5.0.1.tgz#082cb2c89c9fe8659a311a53bd6a4dc5301db304" + integrity sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ== + ansi-styles@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-2.2.1.tgz#b432dd3358b634cf75e1e4664368240533c1ddbe" @@ -3184,6 +3196,13 @@ ansi-styles@^3.0.0, ansi-styles@^3.2.0, ansi-styles@^3.2.1: dependencies: color-convert "^1.9.0" +ansi-styles@^4.1.0: + version "4.3.0" + resolved "https://registry.yarnpkg.com/ansi-styles/-/ansi-styles-4.3.0.tgz#edd803628ae71c04c85ae7a0906edad34b648937" + integrity sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg== + dependencies: + color-convert "^2.0.1" + ansi-to-html@^0.6.6: version "0.6.14" resolved "https://registry.yarnpkg.com/ansi-to-html/-/ansi-to-html-0.6.14.tgz#65fe6d08bba5dd9db33f44a20aec331e0010dad8" @@ -3782,6 +3801,11 @@ async@^1.5.2: resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo= +async@^3.2.4: + version "3.2.4" + resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" + integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== + async@~0.2.9: version "0.2.10" resolved "https://registry.yarnpkg.com/async/-/async-0.2.10.tgz#b6bbe0b0674b9d719708ca38de8c237cb526c3d1" @@ -5713,6 +5737,14 @@ chalk@^2.0.0, chalk@^2.0.1, chalk@^2.1.0, chalk@^2.3.0, chalk@^2.3.1, chalk@^2.4 escape-string-regexp "^1.0.5" supports-color "^5.3.0" +chalk@^4.1.0: + version "4.1.2" + resolved "https://registry.yarnpkg.com/chalk/-/chalk-4.1.2.tgz#aac4e2b7734a740867aeb16bf02aad556a1e7a01" + integrity sha512-oKnbhFyRIXpUuez8iBMmyEa4nbj4IOQyuhc/wy9kY7/WVPcwIO9VA668Pu8RkO7+0G76SLROeyw9CpQ061i4mA== + dependencies: + ansi-styles "^4.1.0" + supports-color "^7.1.0" + chardet@^0.4.0: version "0.4.2" resolved "https://registry.yarnpkg.com/chardet/-/chardet-0.4.2.tgz#b5473b33dc97c424e5d98dc87d55d4d8a29c8bf2" @@ -5823,6 +5855,13 @@ cli-cursor@^2.1.0: dependencies: restore-cursor "^2.0.0" +cli-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/cli-cursor/-/cli-cursor-3.1.0.tgz#264305a7ae490d1d03bf0c9ba7c925d1753af307" + integrity sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw== + dependencies: + restore-cursor "^3.1.0" + cli-spinners@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/cli-spinners/-/cli-spinners-2.1.0.tgz#22c34b4d51f573240885b201efda4e4ec9fff3c7" @@ -5840,6 +5879,11 @@ cli-width@^2.0.0: resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639" integrity sha1-/xnt6Kml5XkyQUewwR8PvLq+1jk= +cli-width@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-3.0.0.tgz#a2f48437a2caa9a22436e794bf071ec9e61cedf6" + integrity sha512-FxqpkPPwu1HjuN93Omfm4h8uIanXofW0RxVEW3k5RKx+mJJYSthzNhp32Kzxxy3YAEZ/Dc/EWN1vZRY0+kOhbw== + cliui@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/cliui/-/cliui-3.2.0.tgz#120601537a916d29940f934da3b48d585a39213d" @@ -5936,11 +5980,23 @@ color-convert@^1.9.0: dependencies: color-name "1.1.3" +color-convert@^2.0.1: + version "2.0.1" + resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-2.0.1.tgz#72d3a68d598c9bdb3af2ad1e84f21d896abd4de3" + integrity sha512-RRECPsj7iu/xb5oKYcsFHSppFNnsj/52OVTRKb4zP5onXwVF3zVmmToNcOfGC+CRDpfK/U584fMg38ZHCaElKQ== + dependencies: + color-name "~1.1.4" + color-name@1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25" integrity sha1-p9BVi9icQveV3UIyj3QIMcpTvCU= +color-name@~1.1.4: + version "1.1.4" + resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2" + integrity sha512-dOy+3AuW3a2wNbZHIuMZpTcgjGuLU/uBL/ubcZF9OXbDo8ff4O8yVp5Bf0efS8uEoYo5q4Fx7dY9OgQGXgAsQA== + color-support@^1.1.3: version "1.1.3" resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" @@ -6443,6 +6499,28 @@ crypto-random-string@^1.0.0: resolved "https://registry.yarnpkg.com/crypto-random-string/-/crypto-random-string-1.0.0.tgz#a230f64f568310e1498009940790ec99545bca7e" integrity sha1-ojD2T1aDEOFJgAmUB5DsmVRbyn4= +csv-parser@^2.3.0: + version "2.3.5" + resolved "https://registry.yarnpkg.com/csv-parser/-/csv-parser-2.3.5.tgz#6b3bf0907684914ff2c5abfbadab111a69eae5db" + integrity sha512-LCHolC4AlNwL+5EuD5LH2VVNKpD8QixZW2zzK1XmrVYUaslFY4c5BooERHOCIubG9iv/DAyFjs4x0HvWNZuyWg== + dependencies: + minimist "^1.2.0" + through2 "^3.0.1" + +csv2json@^2.0.2: + version "2.0.2" + resolved "https://registry.yarnpkg.com/csv2json/-/csv2json-2.0.2.tgz#91702327f0311f26920fd3d54b507f81a3ab9db4" + integrity sha512-YVZ72OehSs9+j2lME10osmJNHeI0dcHlX+qNq8xcR0T+Rt8WaOCchKiEVF3GsK4Of3z/LE64x0ba1cpULmAclw== + dependencies: + csv-parser "^2.3.0" + exec-promise "^0.7.0" + minimist "^1.2.0" + promise-toolbox "^0.14.0" + pump "^3.0.0" + pumpify "^2.0.0" + strip-bom-stream "^4.0.0" + through2 "^3.0.1" + currently-unhandled@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/currently-unhandled/-/currently-unhandled-0.4.1.tgz#988df33feab191ef799a61369dd76c17adf957ea" @@ -6821,6 +6899,16 @@ duplexify@^3.4.2, duplexify@^3.6.0: readable-stream "^2.0.0" stream-shift "^1.0.0" +duplexify@^4.1.1: + version "4.1.2" + resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-4.1.2.tgz#18b4f8d28289132fa0b9573c898d9f903f81c7b0" + integrity sha512-fz3OjcNCHmRP12MJoZMPglx8m4rrFP8rovnk4vT8Fs+aonZoCwGg10dSsQsfP/E62eZcPTMSMP6686fu9Qlqtw== + dependencies: + end-of-stream "^1.4.1" + inherits "^2.0.3" + readable-stream "^3.1.1" + stream-shift "^1.0.0" + each-props@^1.3.0: version "1.3.2" resolved "https://registry.yarnpkg.com/each-props/-/each-props-1.3.2.tgz#ea45a414d16dd5cfa419b1a81720d5ca06892333" @@ -7958,6 +8046,11 @@ emoji-regex@^7.0.1: resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-7.0.3.tgz#933a04052860c85e83c122479c4748a8e4c72156" integrity sha512-CwBLREIQ7LvYFB0WyRvwhq5N5qPhc6PMjD6bYggFlI5YyDgl+0vxq5VHbMOFqLg7hfWzmu8T5Z1QofhmTIhItA== +emoji-regex@^8.0.0: + version "8.0.0" + resolved "https://registry.yarnpkg.com/emoji-regex/-/emoji-regex-8.0.0.tgz#e818fd69ce5ccfcb404594f842963bf53164cc37" + integrity sha512-MSjYzcWNOA0ewAHpz0MxpYFvwg6yjy1NG3xteoqz644VCo/RPgnr1/GGt+ic3iJTzQ8Eu3TdM14SawnVUmGE6A== + emojis-list@^2.0.0: version "2.1.0" resolved "https://registry.yarnpkg.com/emojis-list/-/emojis-list-2.1.0.tgz#4daa4d9db00f9819880c79fa457ae5b09a1fd389" @@ -7982,6 +8075,13 @@ end-of-stream@^1.0.0, end-of-stream@^1.1.0: dependencies: once "^1.4.0" +end-of-stream@^1.4.1: + version "1.4.4" + resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" + integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== + dependencies: + once "^1.4.0" + engine.io-client@~3.3.1: version "3.3.2" resolved "https://registry.yarnpkg.com/engine.io-client/-/engine.io-client-3.3.2.tgz#04e068798d75beda14375a264bb3d742d7bc33aa" @@ -8440,6 +8540,13 @@ evp_bytestokey@^1.0.0, evp_bytestokey@^1.0.3: md5.js "^1.3.4" safe-buffer "^5.1.1" +exec-promise@^0.7.0: + version "0.7.0" + resolved "https://registry.yarnpkg.com/exec-promise/-/exec-promise-0.7.0.tgz#74d55e60c858a94b325e8e20b66a1bd2ae9c184e" + integrity sha512-xrgX4GWiPNsPQrKCJ8X3VObsidav6JrWGFMgmpU2K9TSAu6Q89J+yY6kp/othYRXJ4UhmeO8jBj46ZQRr54XgA== + dependencies: + log-symbols "^1.0.2" + exec-sh@^0.3.2: version "0.3.2" resolved "https://registry.yarnpkg.com/exec-sh/-/exec-sh-0.3.2.tgz#6738de2eb7c8e671d0366aea0b0db8c6f7d7391b" @@ -8694,6 +8801,13 @@ figures@^2.0.0: dependencies: escape-string-regexp "^1.0.5" +figures@^3.0.0: + version "3.2.0" + resolved "https://registry.yarnpkg.com/figures/-/figures-3.2.0.tgz#625c18bd293c604dc4a8ddb2febf0c88341746af" + integrity sha512-yaduQFRKLXYOGgEn6AZau90j3ggSOyiqXU0F9JZfeXYhNa+Jk4X+s45A2zg5jns87GAFa34BBm2kXw4XpNcbdg== + dependencies: + escape-string-regexp "^1.0.5" + file-entry-cache@^5.0.1: version "5.0.1" resolved "https://registry.yarnpkg.com/file-entry-cache/-/file-entry-cache-5.0.1.tgz#ca0f6efa6dd3d561333fb14515065c2fafdf439c" @@ -8823,6 +8937,11 @@ fireworm@^0.7.0: lodash.flatten "^3.0.2" minimatch "^3.0.2" +first-chunk-stream@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/first-chunk-stream/-/first-chunk-stream-3.0.0.tgz#06972a66263505ed82b2c4db93c1b5e078a6576a" + integrity sha512-LNRvR4hr/S8cXXkIY5pTgVP7L3tq6LlYWcg9nWBuW7o1NMxKZo6oOVa/6GIekMGI0Iw7uC+HWimMe9u/VAeKqw== + fixturify-project@^1.10.0: version "1.10.0" resolved "https://registry.yarnpkg.com/fixturify-project/-/fixturify-project-1.10.0.tgz#091c452a9bb15f09b6b9cc7cf5c0ad559f1d9aad" @@ -9654,6 +9773,11 @@ has-flag@^3.0.0: resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-3.0.0.tgz#b5d454dc2199ae225699f3467e5a07f3b955bafd" integrity sha1-tdRU3CGZriJWmfNGfloH87lVuv0= +has-flag@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/has-flag/-/has-flag-4.0.0.tgz#944771fd9c81c81265c4d6941860da06bb59479b" + integrity sha512-EykJT/Q1KjTWctppgIAgfSO0tKVuZUjhgMr17kqTumMl6Afv3EISleU7qZUzoXDFTAHTDC4NOoG/ZxU3EvlMPQ== + has-symbol-support-x@^1.4.1: version "1.4.2" resolved "https://registry.yarnpkg.com/has-symbol-support-x/-/has-symbol-support-x-1.4.2.tgz#1409f98bc00247da45da67cee0a36f282ff26455" @@ -10098,6 +10222,25 @@ inquirer@^6, inquirer@^6.2.0, inquirer@^6.2.2: strip-ansi "^5.1.0" through "^2.3.6" +inquirer@^7.3.3: + version "7.3.3" + resolved "https://registry.yarnpkg.com/inquirer/-/inquirer-7.3.3.tgz#04d176b2af04afc157a83fd7c100e98ee0aad003" + integrity sha512-JG3eIAj5V9CwcGvuOmoo6LB9kbAYT8HXffUl6memuszlwDC/qvFAJw49XJ5NROSFNPxp3iQg1GqkFhaY/CR0IA== + dependencies: + ansi-escapes "^4.2.1" + chalk "^4.1.0" + cli-cursor "^3.1.0" + cli-width "^3.0.0" + external-editor "^3.0.3" + figures "^3.0.0" + lodash "^4.17.19" + mute-stream "0.0.8" + run-async "^2.4.0" + rxjs "^6.6.0" + string-width "^4.1.0" + strip-ansi "^6.0.0" + through "^2.3.6" + interpret@^1.1.0: version "1.2.0" resolved "https://registry.yarnpkg.com/interpret/-/interpret-1.2.0.tgz#d5061a6224be58e8083985f5014d844359576296" @@ -10286,6 +10429,11 @@ is-fullwidth-code-point@^2.0.0: resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-2.0.0.tgz#a3b30a5c4f199183167aaab93beefae3ddfb654f" integrity sha1-o7MKXE8ZkYMWeqq5O+764937ZU8= +is-fullwidth-code-point@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/is-fullwidth-code-point/-/is-fullwidth-code-point-3.0.0.tgz#f116f8064fe90b3f7844a38997c0b75051269f1d" + integrity sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg== + is-git-url@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/is-git-url/-/is-git-url-1.0.0.tgz#53f684cd143285b52c3244b4e6f28253527af66b" @@ -11447,6 +11595,13 @@ lodash@^4.17.19: resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.20.tgz#b44a9b6297bcb698f1c51a3545a2b3b368d59c52" integrity sha512-PlhdFcillOINfeV7Ni6oF1TAEayyZBoZ8bcshTHqOYJYlrqzRK5hagpagky5o4HfCzzd1TRkXPMFq6cKk9rGmA== +log-symbols@^1.0.2: + version "1.0.2" + resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-1.0.2.tgz#376ff7b58ea3086a0f09facc74617eca501e1a18" + integrity sha512-mmPrW0Fh2fxOzdBbFv4g1m6pR72haFLPJ2G5SJEELf1y+iaQrDG6cWCPjy54RHYbZAt7X+ls690Kw62AdWXBzQ== + dependencies: + chalk "^1.0.0" + log-symbols@^2.2.0: version "2.2.0" resolved "https://registry.yarnpkg.com/log-symbols/-/log-symbols-2.2.0.tgz#5740e1c5d6f0dfda4ad9323b5332107ef6b4c40a" @@ -11543,6 +11698,11 @@ make-dir@^2.0.0: pify "^4.0.1" semver "^5.6.0" +make-error@^1.3.2: + version "1.3.6" + resolved "https://registry.yarnpkg.com/make-error/-/make-error-1.3.6.tgz#2eb2e37ea9b67c4891f684a1394799af484cf7a2" + integrity sha512-s8UhlNe7vPKomQhC1qFelMokr/Sc3AgNbso3n74mVPA5LTZwkB9NlXf4XPamLxJE8h0gh73rM94xvwRT2CVInw== + make-fetch-happen@^4.0.1: version "4.0.1" resolved "https://registry.yarnpkg.com/make-fetch-happen/-/make-fetch-happen-4.0.1.tgz#141497cb878f243ba93136c83d8aba12c216c083" @@ -12119,7 +12279,7 @@ mute-stream@0.0.7: resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.7.tgz#3075ce93bc21b8fab43e1bc4da7e8115ed1e7bab" integrity sha1-MHXOk7whuPq0PhvE2n6BFe0ee6s= -mute-stream@~0.0.4: +mute-stream@0.0.8, mute-stream@~0.0.4: version "0.0.8" resolved "https://registry.yarnpkg.com/mute-stream/-/mute-stream-0.0.8.tgz#1630c42b2251ff81e2a283de96a5497ea92e5e0d" integrity sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA== @@ -13317,6 +13477,13 @@ promise-retry@^1.1.1: err-code "^1.0.0" retry "^0.10.0" +promise-toolbox@^0.14.0: + version "0.14.0" + resolved "https://registry.yarnpkg.com/promise-toolbox/-/promise-toolbox-0.14.0.tgz#b2f8bd90fce6709b290b58fc06d89280375a98b4" + integrity sha512-VV5lXK4lXaPB9oBO50ope1qd0AKN8N3nK14jYvV9/qFmfZW2Px/bJjPZBniGjXcIJf6J5Y/coNgJtPHDyiUV/g== + dependencies: + make-error "^1.3.2" + promise.prototype.finally@^3.1.0: version "3.1.0" resolved "https://registry.yarnpkg.com/promise.prototype.finally/-/promise.prototype.finally-3.1.0.tgz#66f161b1643636e50e7cf201dc1b84a857f3864e" @@ -13410,6 +13577,15 @@ pumpify@^1.3.3, pumpify@^1.3.5: inherits "^2.0.3" pump "^2.0.0" +pumpify@^2.0.0: + version "2.0.1" + resolved "https://registry.yarnpkg.com/pumpify/-/pumpify-2.0.1.tgz#abfc7b5a621307c728b551decbbefb51f0e4aa1e" + integrity sha512-m7KOje7jZxrmutanlkS1daj1dS6z6BgslzOXmcSEpIlCxM3VJH7lG5QLeck/6hgF6F4crFf01UtQmNsJfweTAw== + dependencies: + duplexify "^4.1.1" + inherits "^2.0.3" + pump "^3.0.0" + punycode@1.3.2: version "1.3.2" resolved "https://registry.yarnpkg.com/punycode/-/punycode-1.3.2.tgz#9653a036fb7c1ee42342f2325cceefea3926c48d" @@ -14135,6 +14311,14 @@ restore-cursor@^2.0.0: onetime "^2.0.0" signal-exit "^3.0.2" +restore-cursor@^3.1.0: + version "3.1.0" + resolved "https://registry.yarnpkg.com/restore-cursor/-/restore-cursor-3.1.0.tgz#39f67c54b3a7a58cea5236d95cf0034239631f7e" + integrity sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA== + dependencies: + onetime "^5.1.0" + signal-exit "^3.0.2" + ret@~0.1.10: version "0.1.15" resolved "https://registry.yarnpkg.com/ret/-/ret-0.1.15.tgz#b8a4825d5bdb1fc3f6f53c2bc33f81388681c7bc" @@ -14230,6 +14414,11 @@ run-async@^2.2.0: dependencies: is-promise "^2.1.0" +run-async@^2.4.0: + version "2.4.1" + resolved "https://registry.yarnpkg.com/run-async/-/run-async-2.4.1.tgz#8440eccf99ea3e70bd409d49aab88e10c189a455" + integrity sha512-tvVnVv01b8c1RrA6Ep7JkStj85Guv/YrMcwqYQnwjsAS2cTmmPGBBjAjpCW7RrSodNSoE2/qg9O4bceNvUuDgQ== + run-queue@^1.0.0, run-queue@^1.0.3: version "1.0.3" resolved "https://registry.yarnpkg.com/run-queue/-/run-queue-1.0.3.tgz#e848396f057d223f24386924618e25694161ec47" @@ -14256,6 +14445,13 @@ rxjs@^6.4.0: dependencies: tslib "^1.9.0" +rxjs@^6.6.0: + version "6.6.7" + resolved "https://registry.yarnpkg.com/rxjs/-/rxjs-6.6.7.tgz#90ac018acabf491bf65044235d5863c4dab804c9" + integrity sha512-hTdwr+7yYNIT5n4AMYp85KA6yw2Va0FLa3Rguvbpa4W3I5xynaBZo41cM3XM+4Q6fRMj3sBYIR1VAmZMXYJvRQ== + dependencies: + tslib "^1.9.0" + safe-buffer@5.1.2, safe-buffer@^5.0.1, safe-buffer@^5.1.0, safe-buffer@^5.1.1, safe-buffer@^5.1.2, safe-buffer@~5.1.0, safe-buffer@~5.1.1: version "5.1.2" resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d" @@ -14989,6 +15185,15 @@ string-width@^3.0.0: is-fullwidth-code-point "^2.0.0" strip-ansi "^5.1.0" +string-width@^4.1.0: + version "4.2.3" + resolved "https://registry.yarnpkg.com/string-width/-/string-width-4.2.3.tgz#269c7117d27b05ad2e536830a8ec895ef9c6d010" + integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== + dependencies: + emoji-regex "^8.0.0" + is-fullwidth-code-point "^3.0.0" + strip-ansi "^6.0.1" + string_decoder@0.10, string_decoder@~0.10.x: version "0.10.31" resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-0.10.31.tgz#62e203bc41766c6c28c9fc84301dab1c5310fa94" @@ -15029,6 +15234,28 @@ strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" +strip-ansi@^6.0.0, strip-ansi@^6.0.1: + version "6.0.1" + resolved "https://registry.yarnpkg.com/strip-ansi/-/strip-ansi-6.0.1.tgz#9e26c63d30f53443e9489495b2105d37b67a85d9" + integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== + dependencies: + ansi-regex "^5.0.1" + +strip-bom-buf@^2.0.0: + version "2.0.0" + resolved "https://registry.yarnpkg.com/strip-bom-buf/-/strip-bom-buf-2.0.0.tgz#ff9c223937f8e7154b77e9de9bde094186885c15" + integrity sha512-gLFNHucd6gzb8jMsl5QmZ3QgnUJmp7qn4uUSHNwEXumAp7YizoGYw19ZUVfuq4aBOQUtyn2k8X/CwzWB73W2lQ== + dependencies: + is-utf8 "^0.2.1" + +strip-bom-stream@^4.0.0: + version "4.0.0" + resolved "https://registry.yarnpkg.com/strip-bom-stream/-/strip-bom-stream-4.0.0.tgz#4d21a651e723ef743a0a8b0d4534471805330cbb" + integrity sha512-0ApK3iAkHv6WbgLICw/J4nhwHeDZsBxIIsOD+gHgZICL6SeJ0S9f/WZqemka9cjkTyMN5geId6e8U5WGFAn3cQ== + dependencies: + first-chunk-stream "^3.0.0" + strip-bom-buf "^2.0.0" + strip-bom@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/strip-bom/-/strip-bom-2.0.0.tgz#6219a85616520491f35788bdbf1447a99c7e6b0e" @@ -15117,6 +15344,13 @@ supports-color@^5.3.0: dependencies: has-flag "^3.0.0" +supports-color@^7.1.0: + version "7.2.0" + resolved "https://registry.yarnpkg.com/supports-color/-/supports-color-7.2.0.tgz#1b7dcdcb32b8138801b3e478ba6a51caa89648da" + integrity sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw== + dependencies: + has-flag "^4.0.0" + sver-compat@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/sver-compat/-/sver-compat-1.5.0.tgz#3cf87dfeb4d07b4a3f14827bc186b3fd0c645cd8" @@ -15562,6 +15796,11 @@ type-check@~0.3.2: dependencies: prelude-ls "~1.1.2" +type-fest@^0.21.3: + version "0.21.3" + resolved "https://registry.yarnpkg.com/type-fest/-/type-fest-0.21.3.tgz#d260a24b0198436e133fa26a524a6d65fa3b2e37" + integrity sha512-t0rzBq87m3fVcduHDUFhKmyyX+9eo6WQjZvf51Ea/M0Q7+T374Jp1aUiyUl0GKxp8M/OETVHSDvmkyPgvX+X2w== + type-is@^1.6.16, type-is@~1.6.17, type-is@~1.6.18: version "1.6.18" resolved "https://registry.yarnpkg.com/type-is/-/type-is-1.6.18.tgz#4e552cd05df09467dcbc4ef739de89f2cf37c131" From 1cb9f7547538bef535a5991e428c4b4d15965771 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Tue, 1 Nov 2022 14:48:51 -0500 Subject: [PATCH 03/24] Support upserting records --- .gitignore | 2 + services/import/package.json | 1 + services/import/src/batch.js | 37 ++++++++++++++++++ services/import/src/env.js | 10 +++++ services/import/src/index.js | 4 +- services/import/src/mongodb.js | 4 ++ services/import/src/upsert.js | 68 ++++++++++++++++++++++++++++++++- services/import/src/validate.js | 22 +++++++---- yarn.lock | 66 +++++++++++++++++++++++++++++++- 9 files changed, 202 insertions(+), 12 deletions(-) create mode 100644 services/import/src/batch.js create mode 100644 services/import/src/env.js create mode 100644 services/import/src/mongodb.js diff --git a/.gitignore b/.gitignore index 9b700d62..2a9a2de4 100644 --- a/.gitignore +++ b/.gitignore @@ -89,3 +89,5 @@ typings/ .DS_Store services/import/data/**/*.csv +dump +scripts/download.sh diff --git a/services/import/package.json b/services/import/package.json index c5b033ce..0417fe98 100644 --- a/services/import/package.json +++ b/services/import/package.json @@ -17,6 +17,7 @@ "@base-cms/object-path": "^1.9.0", "@identity-x/service-clients": "^1.2.0", "@identity-x/utils": "^1.2.0", + "@parameter1/mongodb": "^1.6.0", "async": "^3.2.4", "csv2json": "^2.0.2", "inquirer": "^7.3.3", diff --git a/services/import/src/batch.js b/services/import/src/batch.js new file mode 100644 index 00000000..cd74cda5 --- /dev/null +++ b/services/import/src/batch.js @@ -0,0 +1,37 @@ +const { log } = console; + +const batch = async ({ + name, + totalCount, + limit, + page = 1, + handler = () => {}, + retriever = () => {}, +} = {}) => { + if (!totalCount) return; + const pages = Math.ceil(totalCount / limit); + const skip = (page - 1) * limit; + log(`Handling batch ${page} of ${pages} (L/S ${limit}/${skip}) for '${name}'`); + + const results = await retriever({ + name, + pages, + page, + limit, + skip, + }); + + await handler({ results, name }); + if (page < pages) { + await batch({ + name, + totalCount, + limit, + page: page + 1, + handler, + retriever, + }); + } +}; + +module.exports = batch; diff --git a/services/import/src/env.js b/services/import/src/env.js new file mode 100644 index 00000000..7e312dce --- /dev/null +++ b/services/import/src/env.js @@ -0,0 +1,10 @@ +const { + cleanEnv, + validators, +} = require('@base-cms/env'); + +const { nonemptystr } = validators; + +module.exports = cleanEnv(process.env, { + MONGO_DSN: nonemptystr({ desc: 'The MongoDB DSN to connect to.' }), +}); diff --git a/services/import/src/index.js b/services/import/src/index.js index 5f4040b1..c91865cd 100644 --- a/services/import/src/index.js +++ b/services/import/src/index.js @@ -66,7 +66,7 @@ const findFilesIn = (path, ext = 'csv', arr = []) => { default: 100, }, { - type: 'boolean', + type: 'confirm', name: 'errorOnBadAnswer', message: 'Should record be skipped if a bad answer value is found?', default: true, @@ -78,7 +78,7 @@ const findFilesIn = (path, ext = 'csv', arr = []) => { log(`Importing records from ${file} to ${appId}!`); const records = await parseCSV(file); const validated = await validate(records, appId, limit, errorOnBadAnswer); - await upsert(validated, appId); + await upsert(validated, appId, limit); log('Import complete!'); } catch (e) { log('Encountered error!', e); diff --git a/services/import/src/mongodb.js b/services/import/src/mongodb.js new file mode 100644 index 00000000..5e971edf --- /dev/null +++ b/services/import/src/mongodb.js @@ -0,0 +1,4 @@ +const Client = require('@parameter1/mongodb/client'); +const { MONGO_DSN } = require('./env'); + +module.exports = new Client({ url: MONGO_DSN }); diff --git a/services/import/src/upsert.js b/services/import/src/upsert.js index 0d761a4e..88fe92a9 100644 --- a/services/import/src/upsert.js +++ b/services/import/src/upsert.js @@ -1,5 +1,69 @@ +const { ObjectId } = require('@parameter1/mongodb'); +const batch = require('./batch'); +const client = require('./mongodb'); + const { log } = console; +const now = new Date(); + +module.exports = async (records = [], appId, limit = 10) => { + log('Upserting ', records.length, appId, limit); + + const applicationId = new ObjectId(appId); + const collection = await client.collection({ dbName: 'identity-x', name: 'app-users' }); + + await batch({ + name: 'upsert', + totalCount: records.length, + limit, + retriever: ({ skip }) => records.slice(skip, skip + limit), + handler: async ({ results }) => { + const ops = results.reduce((arr, user) => { + const { + _id, + email, + verified, + externalId, + customBooleanFieldAnswers, + ...payload + } = user; + const insertDefaults = { + verified, + customBooleanFieldAnswers, + }; + const filter = { applicationId, email, ...(_id && { _id }) }; + const $addToSet = { + ...(externalId && { externalIds: externalId }), + }; + return [ + ...arr, + { + // Upsert the user + updateOne: { + filter, + update: { + ...(Object.keys($addToSet).length && { $addToSet }), + $setOnInsert: { ...insertDefaults, ...filter, _importedAt: now }, + $set: { ...payload, _updatedAt: now }, + }, + upsert: !_id, + }, + }, + { + // Set the boolean answers if they haven't already been set in IdentityX + updateOne: { + filter: { + applicationId, + email, + 'customBooleanFieldAnswers.0': { $exists: false }, + }, + update: { $set: { customBooleanFieldAnswers } }, + }, + }, + ]; + }, []); + return collection.bulkWrite(ops); + }, + }); -module.exports = async (records = [], applicationId, limit = 10) => { - log('Upserting ', records.length, applicationId, limit); + return client.close(); }; diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 0d2d9ff4..e7135724 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -114,11 +114,7 @@ const mapSelectAnswers = async (data, error = false) => { } return [...arr, answerMap.get(value)]; }, []); - // if (values.length !== answers.length) { - // log('v', values, answers); - // process.exit(1); - // } - return { _id: k.id, answers, values }; + return { _id: k.id, values }; }).filter(v => v); }; @@ -177,11 +173,23 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe const { countryName } = filtered; const countryCode = countryName ? await getCountryCode(countryName) : undefined; const validCC = ['US', 'MX', 'CA'].includes(countryCode); + const { _id } = filtered; + if (!isInternal) delete filtered._id; + const email = normalizeEmail(filtered.email); + const [, domain] = email.split('@'); + const externalId = Buffer.from(_id).toString('base64'); const normalized = { ...filtered, - ...(!isInternal && { _id: undefined, externalId: `backoffice.smg.member*${Buffer.from(filtered._id).toString('base64')}` }), + ...(!isInternal && { + externalId: { + // @todo Use util? + _id: `backoffice.smg.member*${externalId}~base64`, + identifier: { value: externalId, type: 'base64' }, + namespace: { provider: 'backoffice', tenant: 'smg', type: 'member' }, + }, + }), email: normalizeEmail(filtered.email), - // domain: + domain, verified: false, receiveEmail: filtered.receiveEmail === 'TRUE', // country is valid/not typo'd diff --git a/yarn.lock b/yarn.lock index df0cf788..f3a942d3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2495,6 +2495,22 @@ universal-user-agent "^2.0.0" url-template "^2.0.8" +"@parameter1/mongodb@^1.6.0": + version "1.6.0" + resolved "https://registry.yarnpkg.com/@parameter1/mongodb/-/mongodb-1.6.0.tgz#9076aac297f4876baf1f8ac3ddd27f6559d88787" + integrity sha512-V6QB8aPWGm+u7S+5u850IkUkWBqHC4MSUYjSNEKSilcWy61fs6Re63X7saIicHjWtMxC+er/hzevbqlyA52Kiw== + dependencies: + "@parameter1/utils" "^1.1.24" + async "^3.2.1" + mongodb "^3.7.2" + +"@parameter1/utils@^1.1.24": + version "1.1.24" + resolved "https://registry.yarnpkg.com/@parameter1/utils/-/utils-1.1.24.tgz#5529765305d22d6698021ecc51d114583e033ae8" + integrity sha512-hRTjEHaDIgEwEg8RJ7IhYhQgfieq2uUEuPxVp5KjC4/KdneCyKLYrMXzFnBSNmZA1j3+Y4dwQOk/cl9KwN+1GA== + dependencies: + object-path "^0.11.8" + "@protobufjs/aspromise@^1.1.1", "@protobufjs/aspromise@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@protobufjs/aspromise/-/aspromise-1.1.2.tgz#9b8b0cc663d669a7d8f6f5d0893a14d348f30fbf" @@ -3801,7 +3817,7 @@ async@^1.5.2: resolved "https://registry.yarnpkg.com/async/-/async-1.5.2.tgz#ec6a61ae56480c0c3cb241c95618e20892f9672a" integrity sha1-7GphrlZIDAw8skHJVhjiCJL5Zyo= -async@^3.2.4: +async@^3.2.1, async@^3.2.4: version "3.2.4" resolved "https://registry.yarnpkg.com/async/-/async-3.2.4.tgz#2d22e00f8cddeb5fde5dd33522b56d1cf569a81c" integrity sha512-iAB+JbDEGXhyIUavoDl9WP/Jj106Kz9DEn1DPgYw5ruDn0e3Wgi3sKFm55sASdGBNOQB8F59d9qQ7deqrHA8wQ== @@ -4569,6 +4585,14 @@ binaryextensions@^2.1.2: resolved "https://registry.yarnpkg.com/binaryextensions/-/binaryextensions-2.3.0.tgz#1d269cbf7e6243ea886aa41453c3651ccbe13c22" integrity sha512-nAihlQsYGyc5Bwq6+EsubvANYGExeJKHDO3RjnvwU042fawQTQfM3Kxn7IHUXQOz4bzfwsGYYHGSvXyW4zOGLg== +bl@^2.2.1: + version "2.2.1" + resolved "https://registry.yarnpkg.com/bl/-/bl-2.2.1.tgz#8c11a7b730655c5d56898cdc871224f40fd901d5" + integrity sha512-6Pesp1w0DEX1N550i/uGV/TqucVL4AM/pgThFSN/Qq9si1/DF9aIHs1BxD8V/QU0HoeHO6cQRTAuYnLPKq1e4g== + dependencies: + readable-stream "^2.3.5" + safe-buffer "^5.1.1" + blank-object@^1.0.1: version "1.0.2" resolved "https://registry.yarnpkg.com/blank-object/-/blank-object-1.0.2.tgz#f990793fbe9a8c8dd013fb3219420bec81d5f4b9" @@ -5440,6 +5464,11 @@ bson@^1.0.1, bson@^1.1.1, bson@~1.1.1: resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.1.tgz#4330f5e99104c4e751e7351859e2d408279f2f13" integrity sha512-jCGVYLoYMHDkOsbwJZBCqwMHyH4c+wzgI9hG7Z6SZJRXWr+x58pdIbm2i9a/jFGCkRJqRUr8eoI7lDWa0hTkxg== +bson@^1.1.4: + version "1.1.6" + resolved "https://registry.yarnpkg.com/bson/-/bson-1.1.6.tgz#fb819be9a60cd677e0853aee4ca712a785d6618a" + integrity sha512-EvVNVeGo4tHxwi8L6bPj3y3itEvStdwvvlojVxxbyYfoaxJ6keLgrTuKdyfEAszFK+H3olzBuafE0yoh0D1gdg== + btoa-lite@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/btoa-lite/-/btoa-lite-1.0.0.tgz#337766da15801210fdd956c22e9c6891ab9d0337" @@ -6736,6 +6765,11 @@ delegates@^1.0.0: resolved "https://registry.yarnpkg.com/delegates/-/delegates-1.0.0.tgz#84c6e159b81904fdca59a0ef44cd870d31250f9a" integrity sha1-hMbhWbgZBP3KWaDvRM2HDTElD5o= +denque@^1.4.1: + version "1.5.1" + resolved "https://registry.yarnpkg.com/denque/-/denque-1.5.1.tgz#07f670e29c9a78f8faecb2566a1e2c11929c5cbf" + integrity sha512-XwE+iZ4D6ZUB7mfYRMb5wByE8L74HCn30FBN7sWnXksWc1LO1bPDl67pBR9o/kC4z/xSNAwkMYcGgqDV3BE3Hw== + depd@1.1.1: version "1.1.1" resolved "https://registry.yarnpkg.com/depd/-/depd-1.1.1.tgz#5783b4e1c459f06fa5ca27f991f3d06e7a310359" @@ -12171,6 +12205,19 @@ mongodb@3.2.5: mongodb-core "3.2.5" safe-buffer "^5.1.2" +mongodb@^3.7.2: + version "3.7.3" + resolved "https://registry.yarnpkg.com/mongodb/-/mongodb-3.7.3.tgz#b7949cfd0adc4cc7d32d3f2034214d4475f175a5" + integrity sha512-Psm+g3/wHXhjBEktkxXsFMZvd3nemI0r3IPsE0bU+4//PnvNWKkzhZcEsbPcYiWqe8XqXJJEg4Tgtr7Raw67Yw== + dependencies: + bl "^2.2.1" + bson "^1.1.4" + denque "^1.4.1" + optional-require "^1.1.8" + safe-buffer "^5.1.2" + optionalDependencies: + saslprep "^1.0.0" + mongoose-legacy-pluralize@1.0.2: version "1.0.2" resolved "https://registry.yarnpkg.com/mongoose-legacy-pluralize/-/mongoose-legacy-pluralize-1.0.2.tgz#3ba9f91fa507b5186d399fb40854bff18fb563e4" @@ -12732,6 +12779,11 @@ object-path@^0.11.4: resolved "https://registry.yarnpkg.com/object-path/-/object-path-0.11.5.tgz#d4e3cf19601a5140a55a16ad712019a9c50b577a" integrity sha512-jgSbThcoR/s+XumvGMTMf81QVBmah+/Q7K7YduKeKVWL7N111unR2d6pZZarSk6kY/caeNxUDyxOvMWyzoU2eg== +object-path@^0.11.8: + version "0.11.8" + resolved "https://registry.yarnpkg.com/object-path/-/object-path-0.11.8.tgz#ed002c02bbdd0070b78a27455e8ae01fc14d4742" + integrity sha512-YJjNZrlXJFM42wTBn6zgOJVar9KFJvzx6sTWDte8sWZF//cnjl0BxHNpfZx+ZffXX63A9q0b1zsFiBX4g4X5KA== + object-visit@^1.0.0: version "1.0.1" resolved "https://registry.yarnpkg.com/object-visit/-/object-visit-1.0.1.tgz#f79c4493af0c5377b59fe39d395e41042dd045bb" @@ -12853,6 +12905,13 @@ optimist@^0.6.1: minimist "~0.0.1" wordwrap "~0.0.2" +optional-require@^1.1.8: + version "1.1.8" + resolved "https://registry.yarnpkg.com/optional-require/-/optional-require-1.1.8.tgz#16364d76261b75d964c482b2406cb824d8ec44b7" + integrity sha512-jq83qaUb0wNg9Krv1c5OQ+58EK+vHde6aBPzLvPPqJm89UQWsvSuFy9X/OSNJnFeSOKo7btE0n8Nl2+nE+z5nA== + dependencies: + require-at "^1.0.6" + optionator@^0.8.1: version "0.8.3" resolved "https://registry.yarnpkg.com/optionator/-/optionator-0.8.3.tgz#84fa1d036fe9d3c7e21d99884b601167ec8fb495" @@ -14160,6 +14219,11 @@ request@^2.87.0, request@^2.88.0: tunnel-agent "^0.6.0" uuid "^3.3.2" +require-at@^1.0.6: + version "1.0.6" + resolved "https://registry.yarnpkg.com/require-at/-/require-at-1.0.6.tgz#9eb7e3c5e00727f5a4744070a7f560d4de4f6e6a" + integrity sha512-7i1auJbMUrXEAZCOQ0VNJgmcT2VOKPRl2YGJwgpHpC9CE91Mv4/4UYIUm4chGJaI381ZDq1JUicFii64Hapd8g== + require-directory@^2.1.1: version "2.1.1" resolved "https://registry.yarnpkg.com/require-directory/-/require-directory-2.1.1.tgz#8c64ad5fd30dab1c976e2344ffe7f792a6a6df42" From e8e84a76d504286022eb6bb0435ace119a12fc16 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Wed, 2 Nov 2022 09:13:44 -0500 Subject: [PATCH 04/24] Add additional profession mappings --- services/import/src/validate.js | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index e7135724..1d5ec5db 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -41,9 +41,23 @@ const oldAnswerMap = new Map([ ['Urgent Care', ['Urgent care']], // Field: Profession + ['Accessioner', []], + ['Association Professional', []], + ['Administrator', []], + ['Distributor', []], + ['Educator', []], + ['Engineer', []], + ['Equipment Service', []], ['Histotechnologist', ['histotechnologist']], + ['Lab Director/Manager', ['Lab director']], ['Lab Technician', ['Lab technician']], + ['Nursing', []], ['Pathologist', ['pathologist']], + ['Pathology Resident', []], + ['System Administrator', []], + ['Systems Analyst', []], + ['Student', []], + ['Vendor', []], // Field: Org Type ['Academic Institution', ['Academic institution']], From fb927ec2f54f39f446478062f14900bea7ca249f Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Wed, 2 Nov 2022 09:42:14 -0500 Subject: [PATCH 05/24] Remove additional terms --- services/import/src/validate.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 1d5ec5db..1e4460ad 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -44,16 +44,21 @@ const oldAnswerMap = new Map([ ['Accessioner', []], ['Association Professional', []], ['Administrator', []], + ['CMO', []], ['Distributor', []], ['Educator', []], ['Engineer', []], ['Equipment Service', []], ['Histotechnologist', ['histotechnologist']], + ['IT Support', []], ['Lab Director/Manager', ['Lab director']], + ['Lab manager', []], ['Lab Technician', ['Lab technician']], + ['Laboratory manager', []], ['Nursing', []], ['Pathologist', ['pathologist']], ['Pathology Resident', []], + ['Phlebotomist', []], ['System Administrator', []], ['Systems Analyst', []], ['Student', []], From ed9fffb42f993a71ff1080dfa6645e1081480efc Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Wed, 2 Nov 2022 09:42:36 -0500 Subject: [PATCH 06/24] Add mappings for renamed/altered terms --- services/import/src/validate.js | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 1e4460ad..1cae5375 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -44,7 +44,11 @@ const oldAnswerMap = new Map([ ['Accessioner', []], ['Association Professional', []], ['Administrator', []], + ['CEO', ['Chief Executive Officer']], // IdX alt + ['CFO', ['Chief Financial Officer']], // IdX alt ['CMO', []], + ['chief marketing officer', ['Chief Marketing Officer']], // IdX alt + ['chief medical officer', ['Chief Medical Officer']], // IdX alt ['Distributor', []], ['Educator', []], ['Engineer', []], @@ -54,7 +58,9 @@ const oldAnswerMap = new Map([ ['Lab Director/Manager', ['Lab director']], ['Lab manager', []], ['Lab Technician', ['Lab technician']], + ['Laboratory director', ['Lab director']], // IdX alt ['Laboratory manager', []], + ['laboratory supervisor', ['Lab supervisor']], // IdX alt ['Nursing', []], ['Pathologist', ['pathologist']], ['Pathology Resident', []], From 1410dc6c64aa1512a6d5866098ae6d6ed9db1d12 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Wed, 2 Nov 2022 10:13:19 -0500 Subject: [PATCH 07/24] Set dates --- services/import/src/upsert.js | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/services/import/src/upsert.js b/services/import/src/upsert.js index 88fe92a9..18327916 100644 --- a/services/import/src/upsert.js +++ b/services/import/src/upsert.js @@ -3,7 +3,7 @@ const batch = require('./batch'); const client = require('./mongodb'); const { log } = console; -const now = new Date(); +const now = new Date('2022-01-01'); module.exports = async (records = [], appId, limit = 10) => { log('Upserting ', records.length, appId, limit); @@ -29,6 +29,8 @@ module.exports = async (records = [], appId, limit = 10) => { const insertDefaults = { verified, customBooleanFieldAnswers, + createdAt: now, + updatedAt: now, }; const filter = { applicationId, email, ...(_id && { _id }) }; const $addToSet = { @@ -42,8 +44,8 @@ module.exports = async (records = [], appId, limit = 10) => { filter, update: { ...(Object.keys($addToSet).length && { $addToSet }), - $setOnInsert: { ...insertDefaults, ...filter, _importedAt: now }, - $set: { ...payload, _updatedAt: now }, + $setOnInsert: { ...insertDefaults, ...filter }, + $set: { ...payload, _importedAt: now }, }, upsert: !_id, }, From 700f29a024d1e9c0a68f39946228ac0f326607ed Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Thu, 17 Nov 2022 12:37:33 -0600 Subject: [PATCH 08/24] Ensure IdentityX questions & answers are ObjectIds --- services/import/src/validate.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 1cae5375..13b36c6f 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -1,4 +1,5 @@ const validator = require('validator'); +const { ObjectId } = require('@parameter1/mongodb'); const { applicationService, localeService } = require('@identity-x/service-clients'); const { normalizeEmail } = require('@identity-x/utils'); // const { eachLimit } = require('async'); @@ -148,10 +149,10 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 } }); answers.edges.forEach(({ node }) => { // eslint-disable-next-line no-underscore-dangle - fieldMap.set(`Custom: ${node.name}`, { id: node._id, type: node._type }); + fieldMap.set(`Custom: ${node.name}`, { id: ObjectId(node._id), type: node._type }); if (node.options) { node.options.forEach((option) => { - answerMap.set(option.label, option._id); + answerMap.set(option.label, ObjectId(option._id)); }); } }); From a71bd04ae8544504d0e1d91630db968f7729031d Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Thu, 17 Nov 2022 12:40:33 -0600 Subject: [PATCH 09/24] Add new constructor to ObjectId classes --- services/import/src/validate.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 13b36c6f..bc7973e3 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -149,10 +149,10 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 } }); answers.edges.forEach(({ node }) => { // eslint-disable-next-line no-underscore-dangle - fieldMap.set(`Custom: ${node.name}`, { id: ObjectId(node._id), type: node._type }); + fieldMap.set(`Custom: ${node.name}`, { id: new ObjectId(node._id), type: node._type }); if (node.options) { node.options.forEach((option) => { - answerMap.set(option.label, ObjectId(option._id)); + answerMap.set(option.label, new ObjectId(option._id)); }); } }); From cdc8c605ce7ec0d583af05b6f37b82dff4ba8468 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Thu, 17 Nov 2022 13:05:28 -0600 Subject: [PATCH 10/24] Add segment to fix previously written oids --- services/import/src/fix-object-ids.js | 51 +++++++++++++++++++++++++++ services/import/src/index.js | 14 ++++++++ 2 files changed, 65 insertions(+) create mode 100644 services/import/src/fix-object-ids.js diff --git a/services/import/src/fix-object-ids.js b/services/import/src/fix-object-ids.js new file mode 100644 index 00000000..8d6f05ed --- /dev/null +++ b/services/import/src/fix-object-ids.js @@ -0,0 +1,51 @@ +const { ObjectId } = require('@parameter1/mongodb'); +const { iterateCursor } = require('@parameter1/mongodb/utils'); + +const batch = require('./batch'); +const client = require('./mongodb'); + +module.exports = async (appId, limit = 10) => { + const applicationId = new ObjectId(appId); + const collection = await client.collection({ dbName: 'identity-x', name: 'app-users' }); + const projection = { customBooleanFieldAnswers: 1, customSelectFieldAnswers: 1 }; + const query = { + applicationId, + $or: [ + { 'customBooleanFieldAnswers.0._id': { $type: 'string' } }, + { 'customSelectFieldAnswers.0._id': { $type: 'string' } }, + ], + }; + + await batch({ + name: 'fix-ids', + totalCount: await collection.countDocuments(query), + limit, + // Explicitly skipping `sort` since we're modifying what we're querying against! + retriever: () => collection.find(query, { limit, projection }), + handler: async ({ results }) => { + const ops = []; + await iterateCursor(results, async (user) => { + ops.push({ + updateOne: { + filter: { _id: user._id }, + update: { + $set: { + customBooleanFieldAnswers: user.customBooleanFieldAnswers.map(({ _id, value }) => ({ + _id: new ObjectId(_id), + value, + })), + customSelectFieldAnswers: user.customSelectFieldAnswers.map(({ _id, values }) => ({ + _id: new ObjectId(_id), + values: (values || []).map(id => new ObjectId(id)), + })), + }, + }, + }, + }); + }); + return collection.bulkWrite(ops); + }, + }); + + return client.close(); +}; diff --git a/services/import/src/index.js b/services/import/src/index.js index c91865cd..7f190dfe 100644 --- a/services/import/src/index.js +++ b/services/import/src/index.js @@ -5,6 +5,7 @@ const { applicationService } = require('@identity-x/service-clients'); const parseCSV = require('./parse-csv'); const validate = require('./validate'); const upsert = require('./upsert'); +const fixObjectIds = require('./fix-object-ids'); const { log } = console; @@ -33,6 +34,7 @@ const findFilesIn = (path, ext = 'csv', arr = []) => { file, limit, errorOnBadAnswer, + fixObjectIdValues, } = await inquirer.prompt([ { type: 'input', @@ -50,6 +52,12 @@ const findFilesIn = (path, ext = 'csv', arr = []) => { }, default: '629bac8439347cfce3861789', // Lab Pulse }, + { + type: 'confirm', + name: 'fixObjectIdValues', + message: 'Should existing question/answer values be converted to ObjectIds?', + default: false, + }, { type: 'list', name: 'file', @@ -73,6 +81,12 @@ const findFilesIn = (path, ext = 'csv', arr = []) => { }, ]); + if (fixObjectIdValues) { + log(`Fixing existing ObjectId values for ${appId}...`); + await fixObjectIds(appId, limit); + log('Done!'); + process.exit(0); + } try { log(`Importing records from ${file} to ${appId}!`); From 38bb710903735d422dc10e263da03fe18392083a Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Thu, 19 Jan 2023 15:47:09 -0600 Subject: [PATCH 11/24] Update validation for Dr. Bicuspid --- services/import/src/validate.js | 100 ++++++++++++++++++++++++++++++-- 1 file changed, 95 insertions(+), 5 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index bc7973e3..5425c78a 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -4,6 +4,7 @@ const { applicationService, localeService } = require('@identity-x/service-clien const { normalizeEmail } = require('@identity-x/utils'); // const { eachLimit } = require('async'); const eachLimit = require('async/eachLimit'); +const { US } = require('../../../services/locale/src/regions'); const { log } = console; const regionMap = new Map(); @@ -146,7 +147,7 @@ const mapSelectAnswers = async (data, error = false) => { module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswer = false) => { const valid = []; - const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 } }); + const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 }, pagination: { limit: 20 } }); answers.edges.forEach(({ node }) => { // eslint-disable-next-line no-underscore-dangle fieldMap.set(`Custom: ${node.name}`, { id: new ObjectId(node._id), type: node._type }); @@ -173,26 +174,115 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(record.countryName === 'Korea, Republic of' && { countryName: 'South Korea' }), ...(record.countryName === 'Macedonia' && { countryName: 'North Macedonia, Republic of' }), ...(record.countryName === 'Rwandese Republic' && { countryName: 'Rwanda' }), + ...(record.countryName === 'Palestinian Territory' && { countryName: 'Palestinian Territory, Occupied' }), + ...(record.countryName === 'Libyan Arab Jamahiriya' && { countryName: 'Libya' }), + ...(['Serbia and Montenegro', 'Yugoslavia'].includes(record.countryName) && { countryName: 'Serbia' }), + ...(record.countryName === 'Irish Republic' && { countryName: 'Ireland' }), + ...(record.countryName === 'Syria' && { countryName: 'Syrian Arab Republic' }), + ...(record.countryName === 'Republic of Maldova' && { countryName: 'Moldova, Republic of' }), + ...(record.countryName === 'Tanzania' && { countryName: 'Tanzania, United Republic of' }), + ...(record.countryName === 'US Virgin Islands' && { countryName: 'Virgin Islands, U.S.' }), + ...(record.countryName === 'Zimbabwe ' && { countryName: 'Zimbabwe' }), + ...(record.countryName === 'Micronesia' && { countryName: 'Micronesia, Federated States of' }), + ...(record.countryName === 'St. Vincent' && { countryName: 'Saint Vincent and the Grenadines' }), + ...(record.countryName === 'Brunei Darusalaam' && { countryName: 'Brunei Darussalam' }), // Regions ...(record.regionName === 'Toscana' && { countryName: 'Italy' }), ...(record.regionName === 'Mexico City' && { regionName: undefined }), - ...(record.regionName === 'DC' && { regionName: 'District of Columbia' }), - // Canada, eh + // This covers all US shortcodes + ...(US[record.regionName] && { regionName: US[record.regionName].name }), + ...(record.regionName === 'Nuevo Leon' && { regionName: 'Nuevo León' }), + ...(record.regionName === 'NUEVO LEON' && { regionName: 'Nuevo León' }), + // Garza García NL, NL = Nuevo León + ...(record.regionName === 'Garza García NL' && { regionName: 'Nuevo León' }), + ...(['Mexico', 'Estado de Mexico'].includes(record.regionName) && { regionName: 'México' }), + ...(['Ciudad de Mexico', 'CIUDAD DE MEXICO'].includes(record.regionName) && { regionName: 'Ciudad de México' }), + ...(record.regionName === 'Wyoming ' && { regionName: 'Wyoming' }), + ...(['Michoacan', 'Michoacan de Ocampo'].includes(record.regionName) && { regionName: 'Michoacán de Ocampo' }), + ...(record.regionName === 'Queretaro de Arteaga' && { regionName: 'Querétaro' }), + ...(record.regionName === 'Yucatan' && { regionName: 'Yucatán' }), + ...(record.regionName === 'Veracruz-Llave' && { regionName: 'Veracruz de Ignacio de la Llave' }), + ...(['jalisco', 'JALISCO'].includes(record.regionName) && { regionName: 'Jalisco' }), + ...(record.regionName === 'SONORA' && { regionName: 'Sonora' }), + ...(record.regionName === 'chihuahua' && { regionName: 'Chihuahua' }), + ...(record.regionName === 'Tamaulipas ' && { regionName: 'Tamaulipas' }), + ...(record.regionName === 'San Luis Potosi' && { regionName: 'San Luis Potosí' }), + ...(record.regionName === 'CHIAPAS' && { regionName: 'Chiapas' }), + // San Andres Cholula is in Puebla Mexio which is the state + ...(record.regionName === 'sn andres cholula' && { regionName: 'Puebla' }), + // Mexicalu B.C. Mexico B.C. is Baja California + ...(record.regionName === 'Mexicalu B.C. Mexico' && { regionName: 'Baja California' }), + ...(record.regionName === 'Coahuila' && { regionName: 'Coahuila de Zaragoza' }), + ...(record.regionName === 'COAHUILA' && { regionName: 'Coahuila de Zaragoza' }), + // Occassionally the Canadians don't see to want to put they're from Canada + ...(record.regionName === 'Alberta' && { countryName: 'Canada' }), + ...(record.regionName === 'NL' && { regionName: 'Newfoundland and Labrador', countryName: 'Canada' }), + ...(record.regionName === 'Ontario' && { countryName: 'Canada' }), + + // Canada, eh? ...(record.countryName === 'Canada' && { ...(record.regionName === 'AB' && { regionName: 'Alberta' }), + ...(record.regionName === 'ab' && { regionName: 'Alberta' }), + ...(record.regionName === 'alberta' && { regionName: 'Alberta' }), ...(record.regionName === 'BC' && { regionName: 'British Columbia' }), + ...(record.regionName === 'bc' && { regionName: 'British Columbia' }), + ...(record.regionName === 'BC - British Columbia' && { regionName: 'British Columbia' }), + ...(record.regionName === 'B.C.' && { regionName: 'British Columbia' }), + ...(record.regionName === 'b.c.' && { regionName: 'British Columbia' }), ...(record.regionName === 'manitoba' && { regionName: 'Manitoba' }), ...(record.regionName === 'NS' && { regionName: 'Nova Scotia' }), ...(record.regionName === 'Qc' && { regionName: 'Quebec' }), + ...(record.regionName === 'qc' && { regionName: 'Quebec' }), + ...(record.regionName === 'quebec' && { regionName: 'Quebec' }), + ...(record.regionName === 'quebec' && { regionName: 'Québec' }), ...(record.regionName === 'Newfoundland' && { regionName: 'Newfoundland and Labrador' }), + ...(record.regionName === 'NL' && { regionName: 'Newfoundland and Labrador' }), ...(record.regionName === 'ONTARIO' && { regionName: 'Ontario' }), + ...(record.regionName === 'ontario' && { regionName: 'Ontario' }), + ...(record.regionName === 'Ontario' && { regionName: 'Ontario' }), + ...(record.regionName === 'on' && { regionName: 'Ontario' }), + ...(record.regionName === 'ON' && { regionName: 'Ontario' }), + ...(record.regionName === 'ont' && { regionName: 'Ontario' }), + ...(record.regionName === 'Ont' && { regionName: 'Ontario' }), + ...(record.regionName === 'otario' && { regionName: 'Ontario' }), + ...(record.regionName === 'On' && { regionName: 'Ontario' }), + ...(record.regionName === 'Yukon Territory', { regionName: 'Yukon' }), + ...(record.regionName === 'sk', { regionName: 'Saskatchewan' }), + ...(record.regionName === 'New-Brunswick' && { regionName: 'New Brunswick' }), }), // Bad data - ...(['AF5B5252903A4', 'CD79C0974D719', 'Choose One'].includes(record.countryName) && { + ...([ + 'AF5B5252903A4', + 'CD79C0974D719', + '318A46E89ABC3', + '759194E4E8096', + 'E8C7BDA5E5AD8', + 'F8E618435CD4D', + 'Choose One', + 'NULL', + '\'null\'', + 'Europe', + 'Anonymous Proxy', + 'Asia/Pacific Region', + 'Satellite Provider', + // Dissolved in 2010 + 'Netherlands Antilles', + ].includes(record.countryName) && { countryName: undefined, }), - ...(['AF5B5252903A4', 'CD79C0974D719', 'Choose One'].includes(record.regionName) && { + ...([ + 'AF5B5252903A4', + 'CD79C0974D719', + 'Choose One', + 'NULL', + '\'null\'', + 'null', + 'Distrito Federal', + 'Mb', + 'Armed Forces Europe, Midd', + 'Armed Forces Europe, Middle East, & Canada', + ].includes(record.regionName) && { regionName: undefined, }), }), {}); From a08566001c788ea56a6868e3bb15a528822c7d56 Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Fri, 20 Jan 2023 08:17:40 -0600 Subject: [PATCH 12/24] Trim region and country names prior to corrections --- services/import/src/validate.js | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 5425c78a..a0e9a084 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -165,7 +165,8 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe try { const filtered = Object.keys(record).reduce((obj, key) => ({ ...obj, - ...(record[key] && /^Custom:/.test(key) === false && { [key]: record[key] }), + ...(record[key] && /^Custom:/.test(key) === false && !['countryName', 'regionName'].includes(key) && { [key]: record[key] }), + ...(record[key] && /^Custom:/.test(key) === false && ['countryName', 'regionName'].includes(key) && { [key]: record[key].trim() }), // Fix bad data ...(record.countryName === 'United States' && { countryName: 'United States of America' }), ...(record.countryName === 'Russia' && { countryName: 'Russian Federation' }), @@ -182,7 +183,6 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(record.countryName === 'Republic of Maldova' && { countryName: 'Moldova, Republic of' }), ...(record.countryName === 'Tanzania' && { countryName: 'Tanzania, United Republic of' }), ...(record.countryName === 'US Virgin Islands' && { countryName: 'Virgin Islands, U.S.' }), - ...(record.countryName === 'Zimbabwe ' && { countryName: 'Zimbabwe' }), ...(record.countryName === 'Micronesia' && { countryName: 'Micronesia, Federated States of' }), ...(record.countryName === 'St. Vincent' && { countryName: 'Saint Vincent and the Grenadines' }), ...(record.countryName === 'Brunei Darusalaam' && { countryName: 'Brunei Darussalam' }), @@ -197,7 +197,6 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(record.regionName === 'Garza García NL' && { regionName: 'Nuevo León' }), ...(['Mexico', 'Estado de Mexico'].includes(record.regionName) && { regionName: 'México' }), ...(['Ciudad de Mexico', 'CIUDAD DE MEXICO'].includes(record.regionName) && { regionName: 'Ciudad de México' }), - ...(record.regionName === 'Wyoming ' && { regionName: 'Wyoming' }), ...(['Michoacan', 'Michoacan de Ocampo'].includes(record.regionName) && { regionName: 'Michoacán de Ocampo' }), ...(record.regionName === 'Queretaro de Arteaga' && { regionName: 'Querétaro' }), ...(record.regionName === 'Yucatan' && { regionName: 'Yucatán' }), @@ -205,7 +204,6 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(['jalisco', 'JALISCO'].includes(record.regionName) && { regionName: 'Jalisco' }), ...(record.regionName === 'SONORA' && { regionName: 'Sonora' }), ...(record.regionName === 'chihuahua' && { regionName: 'Chihuahua' }), - ...(record.regionName === 'Tamaulipas ' && { regionName: 'Tamaulipas' }), ...(record.regionName === 'San Luis Potosi' && { regionName: 'San Luis Potosí' }), ...(record.regionName === 'CHIAPAS' && { regionName: 'Chiapas' }), // San Andres Cholula is in Puebla Mexio which is the state @@ -249,7 +247,8 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(record.regionName === 'Yukon Territory', { regionName: 'Yukon' }), ...(record.regionName === 'sk', { regionName: 'Saskatchewan' }), ...(record.regionName === 'New-Brunswick' && { regionName: 'New Brunswick' }), - }), + } + ), // Bad data ...([ From babfdd7039905e63839e36146cc183a8da046cf6 Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Tue, 31 Jan 2023 12:28:02 -0600 Subject: [PATCH 13/24] Additional validation for custom fields --- services/import/src/validate.js | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index a0e9a084..53ffd63d 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -23,6 +23,11 @@ const oldAnswerMap = new Map([ ['Microscopy & Imaging', ['Pathology and Histology']], ['Molecular Diagnostics', ['Molecular Diagnostics', 'Sequencing', 'Liquid Biopsy', 'Genomics']], // re-route unchanged @todo ['Sequencing', ['Molecular Diagnostics', 'Sequencing', 'Liquid Biopsy', 'Genomics']], // re-route unchanged @todo + ['Regeneration/remineralization', ['Remineralization']], + ['SDF', ['Fluoride/SDF']], + ['Digital Imaging Systems', ['Digital Imaging systems']], + ['Root canal', []], + ['Practice Management Software', []], // Field: Specialities ['Autoimmune Testing', ['Autoimmune']], @@ -71,6 +76,8 @@ const oldAnswerMap = new Map([ ['Systems Analyst', []], ['Student', []], ['Vendor', []], + ['NULL', []], + ['Oral', []], // Field: Org Type ['Academic Institution', ['Academic institution']], From b8e7c5e9f20fd72d1d031a622a95e0e40c6eb2f7 Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Tue, 31 Jan 2023 12:38:28 -0600 Subject: [PATCH 14/24] Account for empty old answer values --- services/import/src/validate.js | 1 + 1 file changed, 1 insertion(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 53ffd63d..34f847ba 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -12,6 +12,7 @@ const countryMap = new Map(); const fieldMap = new Map(); const answerMap = new Map(); const oldAnswerMap = new Map([ + ['', []], // remove empty values // Field: Technologies ['Analyzers & Reagents', ['Molecular Diagnostics', 'Immunoassay', 'Clinical Chemistry']], ['Chromatography', ['Emerging Technologies']], From 46eb2c07e7b955dcd080d17aad802da0f5a3175e Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Tue, 31 Jan 2023 15:24:33 -0600 Subject: [PATCH 15/24] Allow Educator and Student Profession --- services/import/src/validate.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 34f847ba..2b83ce96 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -28,7 +28,7 @@ const oldAnswerMap = new Map([ ['SDF', ['Fluoride/SDF']], ['Digital Imaging Systems', ['Digital Imaging systems']], ['Root canal', []], - ['Practice Management Software', []], + ['Practice Management Software', ['Practice management software']], // Field: Specialities ['Autoimmune Testing', ['Autoimmune']], @@ -58,7 +58,7 @@ const oldAnswerMap = new Map([ ['chief marketing officer', ['Chief Marketing Officer']], // IdX alt ['chief medical officer', ['Chief Medical Officer']], // IdX alt ['Distributor', []], - ['Educator', []], + // ['Educator', []], ['Engineer', []], ['Equipment Service', []], ['Histotechnologist', ['histotechnologist']], @@ -75,7 +75,7 @@ const oldAnswerMap = new Map([ ['Phlebotomist', []], ['System Administrator', []], ['Systems Analyst', []], - ['Student', []], + // ['Student', []], ['Vendor', []], ['NULL', []], ['Oral', []], From 4773c343f6c6676505946376478abeade1929a76 Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Tue, 31 Jan 2023 15:49:54 -0600 Subject: [PATCH 16/24] Point Oral to Oral & Maxillofacial Surgeon --- services/import/src/validate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 2b83ce96..e684cb5e 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -78,7 +78,7 @@ const oldAnswerMap = new Map([ // ['Student', []], ['Vendor', []], ['NULL', []], - ['Oral', []], + ['Oral', ['Oral & Maxillofacial Surgeon']], // Field: Org Type ['Academic Institution', ['Academic institution']], From e9d10e0373e553d76c7a8186383ec15440b0fb52 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Fri, 10 Mar 2023 11:52:27 -0600 Subject: [PATCH 17/24] add email to log --- services/import/src/validate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index e684cb5e..7967249b 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -329,7 +329,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe // log(normalized); valid.push(normalized); } catch (e) { - log('record failed', e); + log('record failed', record.email, e); // throw e; } }); From fbd041d5528294ce82689fb8f57d17247e731473 Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Thu, 6 Apr 2023 16:46:04 -0500 Subject: [PATCH 18/24] Update validation for Aunt Minnie --- services/import/src/validate.js | 351 +++++++++++++++++++++++++++++--- 1 file changed, 323 insertions(+), 28 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index e684cb5e..8f5049a3 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -29,6 +29,16 @@ const oldAnswerMap = new Map([ ['Digital Imaging Systems', ['Digital Imaging systems']], ['Root canal', []], ['Practice Management Software', ['Practice management software']], + ['3D Imaging', ['3D Imaging/3D Printing']], + ['Artificial Intelligence', ['Artificial Intelligence/Computer-aided detection or diagnosis']], + ['Computer-aided detection', []], + ['Interventional Radiology', []], + ['PACS/Teleradiology', ['PACS/Teleradiology/Enterprise Imaging']], + ['Virtual Colonoscopy', []], + ['Digital radiography', ['Digital Radiography']], + ['Information Systems', ['RIS/Information Systems']], + ['& printing systems', ['Film & Film Printing Systems']], + ['Artificial Intelligence/ or diagnosis', ['Artificial Intelligence/Computer-aided detection or diagnosis']], // Field: Specialities ['Autoimmune Testing', ['Autoimmune']], @@ -47,6 +57,9 @@ const oldAnswerMap = new Map([ ['Sexually transmitted diseases', []], // remove ['Substance Abuse Testing', ['Drugs of Abuse/Toxicology']], ['Urgent Care', ['Urgent care']], + ['Information systems subspecialty', ['Imaging Informatics']], + ['PACS', []], + ['Practice management news - subspecialty', ['Practice Management']], // Field: Profession ['Accessioner', []], @@ -62,7 +75,7 @@ const oldAnswerMap = new Map([ ['Engineer', []], ['Equipment Service', []], ['Histotechnologist', ['histotechnologist']], - ['IT Support', []], + ['IT Support', ['IT Professional']], ['Lab Director/Manager', ['Lab director']], ['Lab manager', []], ['Lab Technician', ['Lab technician']], @@ -79,6 +92,8 @@ const oldAnswerMap = new Map([ ['Vendor', []], ['NULL', []], ['Oral', ['Oral & Maxillofacial Surgeon']], + ['MIS Director', []], + ['PACS/RIS Manager', []], // Field: Org Type ['Academic Institution', ['Academic institution']], @@ -93,7 +108,7 @@ const oldAnswerMap = new Map([ ]); const badAnswers = []; -const getRegionCode = async (name, countryCode) => { +const getRegionCode = async ({ name, countryCode }) => { if (regionMap.has(name)) return regionMap.get(name); const code = await localeService.request('region.getCode', { name, countryCode }); @@ -102,8 +117,16 @@ const getRegionCode = async (name, countryCode) => { regionMap.set(name, code); return code; } + const retry = await localeService.request('region.getCode', { + name: name.toLowerCase().split('').map((char, index) => (index ? char : char.toUpperCase())).join(''), + countryCode, + }); + if (retry) { + regionMap.set(name, retry); + return retry; + } // log('resp', code); - throw new Error(`unknown region name "${name}"`); + throw new Error(`unknown region name "${name}" for ${countryCode}`); }; const getCountryCode = async (name) => { @@ -155,7 +178,7 @@ const mapSelectAnswers = async (data, error = false) => { module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswer = false) => { const valid = []; - const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 }, pagination: { limit: 20 } }); + const answers = await applicationService.request('field.listForApp', { id: applicationId, sort: { _id: 1 }, pagination: { limit: 100 } }); answers.edges.forEach(({ node }) => { // eslint-disable-next-line no-underscore-dangle fieldMap.set(`Custom: ${node.name}`, { id: new ObjectId(node._id), type: node._type }); @@ -194,32 +217,246 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(record.countryName === 'Micronesia' && { countryName: 'Micronesia, Federated States of' }), ...(record.countryName === 'St. Vincent' && { countryName: 'Saint Vincent and the Grenadines' }), ...(record.countryName === 'Brunei Darusalaam' && { countryName: 'Brunei Darussalam' }), + ...(record.countryName === 'St.Lucia' && { countryName: 'Saint Lucia' }), + ...(record.countryName === 'British Virgin Islands' && { countryName: 'Virgin Islands, British' }), + ...([ + 'Ivory Coast', + 'Côte d’Ivoire', + ].includes(record.countryName) && { countryName: 'Cote D\'Ivoire' }), + ...(record.countryName === 'Palestine' && { countryName: 'Palestinian Territory, Occupied' }), + ...(record.countryName === 'Turks & Caicos Islands' && { countryName: 'Turks and Caicos Islands' }), + // USSR name + ...(record.countryName === 'Byelorussian' && { countryName: 'Belarus' }), + ...(record.countryName === 'Kiribati Republic' && { countryName: 'Kiribati' }), + // Ruled by the kingdom of Denmark + ...(record.countryName === 'Faeroe Islands' && { countryName: 'Denmark' }), + ...(record.countryName === 'Congo, Dem. Rep. of' && { countryName: 'Congo, the Democratic Republic of the' }), + ...(record.countryName === 'Saint Pierre et Miquelon' && { countryName: 'Saint Pierre and Miquelon' }), + ...(record.countryName === 'Togolese Republic' && { countryName: 'Togo' }), + ...(record.countryName === 'Cocos Islands' && { countryName: 'Cocos (Keeling) Islands' }), + ...([ + 'St. Kitts and Nevis', + 'St.Kitts and Nevis', + ].includes(record.countryName) && { countryName: 'Saint Kitts and Nevis' }), + ...(record.countryName === 'Laos' && { countryName: 'Lao People\'s Democratic Republic' }), + ...(record.countryName === 'Sao Tome e Principe' && { countryName: 'Sao Tome and Principe' }), + ...(record.countryName === 'East Timor' && { countryName: 'Timor-Leste' }), + ...(record.countryName === 'Hong Kong SAR China' && { countryName: 'Hong Kong' }), + ...(record.countryName === 'Falkland Islands' && { countryName: 'Falkland Islands (Malvinas)' }), + ...(record.countryName === 'Western Samoa' && { countryName: 'Samoa' }), + + // Regions ...(record.regionName === 'Toscana' && { countryName: 'Italy' }), - ...(record.regionName === 'Mexico City' && { regionName: undefined }), + ...(record.regionName === 'Distrito Federal' && { + countryName: 'Mexico', + regionName: 'Ciudad de México', + }), + ...(record.regionName === 'Quebec' && { countryName: 'Canada', regionName: 'Quebec' }), + ...(record.regionName === 'London, City of' && { regionName: undefined }), + ...(record.regionName === 'Mexico City' && record.countryName !== 'Mexico' && { + regionName: undefined, + }), + ...(record.regionName === 'dc' && { regionName: 'District of Columbia' }), // This covers all US shortcodes ...(US[record.regionName] && { regionName: US[record.regionName].name }), - ...(record.regionName === 'Nuevo Leon' && { regionName: 'Nuevo León' }), - ...(record.regionName === 'NUEVO LEON' && { regionName: 'Nuevo León' }), - // Garza García NL, NL = Nuevo León - ...(record.regionName === 'Garza García NL' && { regionName: 'Nuevo León' }), - ...(['Mexico', 'Estado de Mexico'].includes(record.regionName) && { regionName: 'México' }), - ...(['Ciudad de Mexico', 'CIUDAD DE MEXICO'].includes(record.regionName) && { regionName: 'Ciudad de México' }), - ...(['Michoacan', 'Michoacan de Ocampo'].includes(record.regionName) && { regionName: 'Michoacán de Ocampo' }), - ...(record.regionName === 'Queretaro de Arteaga' && { regionName: 'Querétaro' }), - ...(record.regionName === 'Yucatan' && { regionName: 'Yucatán' }), - ...(record.regionName === 'Veracruz-Llave' && { regionName: 'Veracruz de Ignacio de la Llave' }), - ...(['jalisco', 'JALISCO'].includes(record.regionName) && { regionName: 'Jalisco' }), - ...(record.regionName === 'SONORA' && { regionName: 'Sonora' }), - ...(record.regionName === 'chihuahua' && { regionName: 'Chihuahua' }), - ...(record.regionName === 'San Luis Potosi' && { regionName: 'San Luis Potosí' }), - ...(record.regionName === 'CHIAPAS' && { regionName: 'Chiapas' }), - // San Andres Cholula is in Puebla Mexio which is the state - ...(record.regionName === 'sn andres cholula' && { regionName: 'Puebla' }), - // Mexicalu B.C. Mexico B.C. is Baja California - ...(record.regionName === 'Mexicalu B.C. Mexico' && { regionName: 'Baja California' }), - ...(record.regionName === 'Coahuila' && { regionName: 'Coahuila de Zaragoza' }), - ...(record.regionName === 'COAHUILA' && { regionName: 'Coahuila de Zaragoza' }), + ...(record.regionName === 'west virginia' && { regionName: 'West Virginia' }), + ...(record.regionName === 'new york' && { regionName: 'New York' }), + + ...(record.countryName === 'Mexico' && { + ...([ + 'Garza García NL', + 'Nuevo Leon', + 'NUEVO LEON', + 'nuevo leon', + 'Nuevo leon', + 'Nuevo Le�n', + 'NUEVO LEÓN', + 'nuevo león', + 'Monterrey', + 'MONTERREY', + 'monterrey', + 'San nicolas de los garza', + ].includes(record.regionName) && { regionName: 'Nuevo León' }), + ...([ + 'Mexico', + 'Estado de Mexico', + 'La Paz. Edo. de Mexico', + 'Estado de M�xic', + 'Estado de M�xico', + 'MEXICO', + 'estado de mexico', + 'Atizapan', + 'mexico', + 'Coacalco, Edo de México', + 'ESTADO DE MÉXICO', + 'Estado de M?xico', + 'ESTADO DE MEXICO', + 'edo. mexico', + 'La Paz', + 'Estado de México', + 'Zumpango', + 'toluca', + 'TOLUCA', + ].includes(record.regionName) && { regionName: 'México' }), + // Apparently in 2016 this got changed to Ciudad de México from Distrito Federal + ...([ + 'Ciudad de mexico', + 'ciudad de mexico', + 'Ciudad De Mexico', + 'Ciudad de Mexico', + 'CIUDAD DE MEXICO', + 'Alvaro Obregon', + 'IZTAPALAPA.', + 'Distrito Federal', + 'D:F.', + 'MEXICO D.F.', + 'Cuauhtémoc', + 'Cuauhtemoc', + 'CDMX', + 'Cdmx', + 'DF', + 'Mexico D. F.', + 'D.F.', + 'Delegación Tlalpan', + 'd.f', + 'México D.F.', + 'df', + 'Tlalpan', + 'DISTRITO FEDERAL', + 'MEXICO CITY', + 'Distrito federal', + 'Mexico City', + 'd.f.', + 'Mexico city', + 'mexico city', + 'distrtito federal', + 'distrito federal', + 'D. F.', + 'cdmx', + 'M�xico, Distrito Federal', + 'CDMX', + 'M�xico D.F.', + 'D.F', + 'Federal District of Mexico', + ].includes(record.regionName) && { regionName: 'Ciudad de México' }), + ...([ + 'Michoacan', + 'Michoacan de Ocampo', + 'MICHOACAN', + 'Michoacán', + 'Morelia', + 'Mich', + 'michoacán', + 'Mi hoacan', + ].includes(record.regionName) && { regionName: 'Michoacán de Ocampo' }), + ...([ + 'Queretaro de Arteaga', + 'Queretaro', + 'Quer�taro', + 'QUERETARO', + 'Juriquilla', + ].includes(record.regionName) && { regionName: 'Querétaro' }), + ...([ + 'Yucatan', + 'Merida', + 'MERIDA, YUCATA', + 'Merida, yucatan', + 'MERIDA, YUCATAN', + 'Cancún', + ].includes(record.regionName) && { regionName: 'Yucatán' }), + ...([ + 'Veracruz-Llave', + 'Veracruz', + 'veracruz', + 'VERACRUZ', + 'Ver.', + 'BOCA DEL RIO', + ].includes(record.regionName) && { regionName: 'Veracruz de Ignacio de la Llave' }), + ...([ + 'jalisco', + 'JALISCO', + 'JAL', + 'guadalajara', + 'ZAPOPAN', + 'Guadalajara', + ].includes(record.regionName) && { regionName: 'Jalisco' }), + ...([ + 'SONORA', + 'HERMOSILLO', + 'SON', + 'Hermosillo', + ].includes(record.regionName) && { regionName: 'Sonora' }), + ...(record.regionName === 'chihuahua' && { regionName: 'Chihuahua' }), + ...([ + 'San Luis Potosi', + 'san luis potosi', + 'sanluispotosi', + 'SAN LUIS POTOSI', + ].includes(record.regionName) && { regionName: 'San Luis Potosí' }), + ...([ + 'CHIAPAS', + 'HUIXTLA CHIAPAS MEXICO.', + ].includes(record.regionName) && { regionName: 'Chiapas' }), + ...(record.regionName === 'OAXACA' && { regionName: 'Oaxaca' }), + ...([ + 'guanajuato', + 'gto', + 'Gto', + ].includes(record.regionName) && { regionName: 'Guanajuato' }), + ...([ + 'GUERRERO', + 'Acapulco', + 'Gro.', + ].includes(record.regionName) && { regionName: 'Guerrero' }), + // Appears this is the capital of the Hidalgo state in Mexico + ...(record.regionName === 'Pachuca' && { regionName: 'Hidalgo' }), + ...(['CULIACAN', 'SI'].includes(record.regionName) && { regionName: 'Sinaloa' }), + // Common practice to refer to the city based on who it was named after + // Juarez is in the state of Chihuahua + ...([ + 'Benito Juarez', + 'benito juarez', + 'Benito Juárez', + 'CD. JUAREZ', + ].includes(record.regionName) && { regionName: 'Chihuahua' }), + // San Andres Cholula is in Puebla Mexio which is the state + ...([ + 'sn andres cholula', + 'PUEBLA', + 'ZacatlánPue.', + ].includes(record.regionName) && { regionName: 'Puebla' }), + // Mexicalu B.C. Mexico B.C. is Baja California + ...([ + 'Mexicalu B.C. Mexico', + 'baja california', + 'Mexicali', + 'BAJA CALIFORNIA', + 'TIJUANA', + 'Ensenada México', + 'baja cal.', + 'Baja Calif', + 'BC', + 'BAJA CALIFORNIA SUR', + ].includes(record.regionName) && { regionName: 'Baja California' }), + ...([ + 'Coahuila', + 'coahuila', + 'COAHUILA', + 'Torreón', + 'COahuila', + 'Torreon', + 'Monclova', + ].includes(record.regionName) && { regionName: 'Coahuila de Zaragoza' }), + ...(record.regionName === 'QUINTANA ROO' && { regionName: 'Quintana Roo' }), + ...([ + 'tams', + 'tam', + 'Tam', + 'matamoros', + 'TAMPICO', + ].includes(record.regionName) && { regionName: 'Tamaulipas' }), + }), // Occassionally the Canadians don't see to want to put they're from Canada ...(record.regionName === 'Alberta' && { countryName: 'Canada' }), ...(record.regionName === 'NL' && { regionName: 'Newfoundland and Labrador', countryName: 'Canada' }), @@ -275,6 +512,9 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'Satellite Provider', // Dissolved in 2010 'Netherlands Antilles', + 'French Antilles', + // Someone's name is in the wrong field + 'Diego Garcia', ].includes(record.countryName) && { countryName: undefined, }), @@ -285,10 +525,65 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'NULL', '\'null\'', 'null', - 'Distrito Federal', 'Mb', 'Armed Forces Europe, Midd', 'Armed Forces Europe, Middle East, & Canada', + 'Armed Forces Pacific', + 'Armed Forces Americas', + '4186511', + // Could be a city/state in the U.S., Mexico or Yemen + 'San', + // In Brazil + 'Sao Paulo', + // In India + 'Tamil Nadu', + '53280', + // I don't want to know + '________', + // Appears to be a town in the U.K + 'Slough', + '6441093412', + '5515054100', + // Is a country + 'GUATEMALA', + '7291619463', + '123', + '52-55-4390-4729', + // In Ethiopia + 'Adis Abeba', + '*', + // In Colombia + 'Narino', + // In India + 'kerala', + '4448129446', + // In Chile + 'Araucania', + // In Indonesia + 'Jakarta Raya', + // Potentially somewhere in Mexico but unsure where + 'SOLTERO(A)', + // In Slovenia + 'Maribor', + // In Iran + 'Lorestan', + 'Kordestan', + // In Philippines + 'Quezon City', + '* Other', + 'N/A', + 'Selecciona un estado', + 'na', + 'Non us', + 'Centre', + // In China + 'Shanghai', + // In Netherlands + 'Noord-Holland', + 'N / A (N / A)', + 'i', + 'no', + 'T\'ai-wan', ].includes(record.regionName) && { regionName: undefined, }), @@ -319,7 +614,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(countryName && { countryCode }), // region/state codes are set properly ...(filtered.regionName && validCC && { - regionCode: await getRegionCode(filtered.regionName, countryCode), + regionCode: await getRegionCode({ name: filtered.regionName, countryCode }), }), // values for custom questions map to valid answers customBooleanFieldAnswers: await mapBooleanAnswers(record), From c0b82a65bb2000835abf4ca8ae13e98448d3cc9b Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Thu, 6 Apr 2023 16:58:23 -0500 Subject: [PATCH 19/24] Validate on '1' vs 'TRUE' --- services/import/src/validate.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 8f5049a3..be3dbc0b 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -146,7 +146,7 @@ const mapBooleanAnswers = async (data) => { .filter(key => fieldMap.get(key).type === 'boolean'); return keys.map((key) => { const k = fieldMap.get(key); - const value = data[key] === 'TRUE'; + const value = data[key] === '1'; return { _id: k.id, value }; }).filter(v => v); }; From 15f034f6caf4ed763067881a1232699ad899353d Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Thu, 6 Apr 2023 17:26:42 -0500 Subject: [PATCH 20/24] Validation updates --- services/import/src/validate.js | 55 ++++++++++++++++++++++++++++----- 1 file changed, 47 insertions(+), 8 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index 7967249b..e919be67 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -120,10 +120,10 @@ const getCountryCode = async (name) => { const mapBooleanAnswers = async (data) => { const keys = Object.keys(data) .filter(key => /^Custom:/i.test(key)) - .filter(key => fieldMap.get(key).type === 'boolean'); + .filter(key => fieldMap.has(key) && fieldMap.get(key).type === 'boolean'); return keys.map((key) => { const k = fieldMap.get(key); - const value = data[key] === 'TRUE'; + const value = ['TRUE', '1'].includes(data[key]); return { _id: k.id, value }; }).filter(v => v); }; @@ -132,7 +132,7 @@ const mapSelectAnswers = async (data, error = false) => { const keys = Object.keys(data) .filter(key => /^Custom:/i.test(key)) .filter(key => data[key]) - .filter(key => fieldMap.get(key).type === 'select'); + .filter(key => fieldMap.has(key) && fieldMap.get(key).type === 'select'); return keys.map((key) => { const k = fieldMap.get(key); const answers = `${data[key]}`.split('|'); @@ -142,9 +142,10 @@ const mapSelectAnswers = async (data, error = false) => { return [...arr, ...oldAnswerMap.get(value).map(a => answerMap.get(a))]; } if (!answerMap.has(value)) { - badAnswers[key] = badAnswers[key] || new Set(); - badAnswers[key].add(value); - if (error) throw new Error(`Unable to find mapped answer for "${value}"!`); + badAnswers[key] = badAnswers[key] || new Map(); + const bc = badAnswers[key].has(value) ? badAnswers[key].get(value) : 0; + badAnswers[key].set(value, bc + 1); + if (error) throw new Error(`Unable to find mapped answer "${value}" for "${key}"!`); return arr; } return [...arr, answerMap.get(value)]; @@ -296,13 +297,18 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe const { countryName } = filtered; const countryCode = countryName ? await getCountryCode(countryName) : undefined; const validCC = ['US', 'MX', 'CA'].includes(countryCode); + if (!filtered._id) throw new Error('Missing `_id` column, verify CSV!'); const { _id } = filtered; if (!isInternal) delete filtered._id; const email = normalizeEmail(filtered.email); const [, domain] = email.split('@'); const externalId = Buffer.from(_id).toString('base64'); const normalized = { - ...filtered, + // Null out nullish values + ...(Object.keys(filtered).reduce((obj, key) => { + const v = filtered[key]; + return { ...obj, [key]: ['null', 'NULL', '\''].includes(v) ? null : v }; + }, {})), ...(!isInternal && { externalId: { // @todo Use util? @@ -326,7 +332,40 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe customSelectFieldAnswers: await mapSelectAnswers(record, errorOnBadAnswer), }; if (!validator.isEmail(normalized.email)) throw new Error(`${normalized.email} is not a valid email address.`); - // log(normalized); + + // Manually remove extra data + delete normalized.last_email_opened; + delete normalized.date_of_last_session; + + const allowedFields = [ + 'email', + 'domain', + 'verified', + 'receiveEmail', + 'givenName', + 'familyName', + 'organization', + 'organizationTitle', + 'city', + 'regionCode', + 'regionName', + 'postalCode', + 'phone', + 'countryName', + 'countryCode', + 'customSelectFieldAnswers', + 'customBooleanFieldAnswers', + 'externalId', + ...fieldMap.keys(), + ]; + + const extraFields = Object.keys(normalized).reduce((arr, key) => ([ + ...arr, + ...(allowedFields.includes(key) ? [] : [key]), + ]), []); + + if (extraFields.length) throw new Error(`Unexpected extra fields: ${extraFields}`); + valid.push(normalized); } catch (e) { log('record failed', record.email, e); From f95b45cba4233bce589615ad7cbcbfe1a2974da9 Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Fri, 7 Apr 2023 07:53:24 -0500 Subject: [PATCH 21/24] Trim the regionName prior to filtering --- services/import/src/validate.js | 129 ++++++++++++++++---------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index a71d6fac..d5d75a4a 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -194,6 +194,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe await eachLimit(records, limit, async (record) => { const isInternal = /[a-f0-9]{24,}/i.test(record._id); + const regionNameTrimmed = record.regionName.trim(); try { const filtered = Object.keys(record).reduce((obj, key) => ({ ...obj, @@ -248,21 +249,21 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe // Regions - ...(record.regionName === 'Toscana' && { countryName: 'Italy' }), - ...(record.regionName === 'Distrito Federal' && { + ...(regionNameTrimmed === 'Toscana' && { countryName: 'Italy' }), + ...(regionNameTrimmed === 'Distrito Federal' && { countryName: 'Mexico', regionName: 'Ciudad de México', }), - ...(record.regionName === 'Quebec' && { countryName: 'Canada', regionName: 'Quebec' }), - ...(record.regionName === 'London, City of' && { regionName: undefined }), - ...(record.regionName === 'Mexico City' && record.countryName !== 'Mexico' && { + ...(regionNameTrimmed === 'Quebec' && { countryName: 'Canada', regionName: 'Quebec' }), + ...(regionNameTrimmed === 'London, City of' && { regionName: undefined }), + ...(regionNameTrimmed === 'Mexico City' && record.countryName !== 'Mexico' && { regionName: undefined, }), - ...(record.regionName === 'dc' && { regionName: 'District of Columbia' }), + ...(regionNameTrimmed === 'dc' && { regionName: 'District of Columbia' }), // This covers all US shortcodes - ...(US[record.regionName] && { regionName: US[record.regionName].name }), - ...(record.regionName === 'west virginia' && { regionName: 'West Virginia' }), - ...(record.regionName === 'new york' && { regionName: 'New York' }), + ...(US[regionNameTrimmed] && { regionName: US[regionNameTrimmed].name }), + ...(regionNameTrimmed === 'west virginia' && { regionName: 'West Virginia' }), + ...(regionNameTrimmed === 'new york' && { regionName: 'New York' }), ...(record.countryName === 'Mexico' && { ...([ @@ -278,7 +279,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'MONTERREY', 'monterrey', 'San nicolas de los garza', - ].includes(record.regionName) && { regionName: 'Nuevo León' }), + ].includes(regionNameTrimmed) && { regionName: 'Nuevo León' }), ...([ 'Mexico', 'Estado de Mexico', @@ -299,7 +300,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'Zumpango', 'toluca', 'TOLUCA', - ].includes(record.regionName) && { regionName: 'México' }), + ].includes(regionNameTrimmed) && { regionName: 'México' }), // Apparently in 2016 this got changed to Ciudad de México from Distrito Federal ...([ 'Ciudad de mexico', @@ -340,7 +341,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'M�xico D.F.', 'D.F', 'Federal District of Mexico', - ].includes(record.regionName) && { regionName: 'Ciudad de México' }), + ].includes(regionNameTrimmed) && { regionName: 'Ciudad de México' }), ...([ 'Michoacan', 'Michoacan de Ocampo', @@ -350,14 +351,14 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'Mich', 'michoacán', 'Mi hoacan', - ].includes(record.regionName) && { regionName: 'Michoacán de Ocampo' }), + ].includes(regionNameTrimmed) && { regionName: 'Michoacán de Ocampo' }), ...([ 'Queretaro de Arteaga', 'Queretaro', 'Quer�taro', 'QUERETARO', 'Juriquilla', - ].includes(record.regionName) && { regionName: 'Querétaro' }), + ].includes(regionNameTrimmed) && { regionName: 'Querétaro' }), ...([ 'Yucatan', 'Merida', @@ -365,7 +366,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'Merida, yucatan', 'MERIDA, YUCATAN', 'Cancún', - ].includes(record.regionName) && { regionName: 'Yucatán' }), + ].includes(regionNameTrimmed) && { regionName: 'Yucatán' }), ...([ 'Veracruz-Llave', 'Veracruz', @@ -373,7 +374,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'VERACRUZ', 'Ver.', 'BOCA DEL RIO', - ].includes(record.regionName) && { regionName: 'Veracruz de Ignacio de la Llave' }), + ].includes(regionNameTrimmed) && { regionName: 'Veracruz de Ignacio de la Llave' }), ...([ 'jalisco', 'JALISCO', @@ -381,38 +382,38 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'guadalajara', 'ZAPOPAN', 'Guadalajara', - ].includes(record.regionName) && { regionName: 'Jalisco' }), + ].includes(regionNameTrimmed) && { regionName: 'Jalisco' }), ...([ 'SONORA', 'HERMOSILLO', 'SON', 'Hermosillo', - ].includes(record.regionName) && { regionName: 'Sonora' }), - ...(record.regionName === 'chihuahua' && { regionName: 'Chihuahua' }), + ].includes(regionNameTrimmed) && { regionName: 'Sonora' }), + ...(regionNameTrimmed === 'chihuahua' && { regionName: 'Chihuahua' }), ...([ 'San Luis Potosi', 'san luis potosi', 'sanluispotosi', 'SAN LUIS POTOSI', - ].includes(record.regionName) && { regionName: 'San Luis Potosí' }), + ].includes(regionNameTrimmed) && { regionName: 'San Luis Potosí' }), ...([ 'CHIAPAS', 'HUIXTLA CHIAPAS MEXICO.', - ].includes(record.regionName) && { regionName: 'Chiapas' }), - ...(record.regionName === 'OAXACA' && { regionName: 'Oaxaca' }), + ].includes(regionNameTrimmed) && { regionName: 'Chiapas' }), + ...(regionNameTrimmed === 'OAXACA' && { regionName: 'Oaxaca' }), ...([ 'guanajuato', 'gto', 'Gto', - ].includes(record.regionName) && { regionName: 'Guanajuato' }), + ].includes(regionNameTrimmed) && { regionName: 'Guanajuato' }), ...([ 'GUERRERO', 'Acapulco', 'Gro.', - ].includes(record.regionName) && { regionName: 'Guerrero' }), + ].includes(regionNameTrimmed) && { regionName: 'Guerrero' }), // Appears this is the capital of the Hidalgo state in Mexico - ...(record.regionName === 'Pachuca' && { regionName: 'Hidalgo' }), - ...(['CULIACAN', 'SI'].includes(record.regionName) && { regionName: 'Sinaloa' }), + ...(regionNameTrimmed === 'Pachuca' && { regionName: 'Hidalgo' }), + ...(['CULIACAN', 'SI'].includes(regionNameTrimmed) && { regionName: 'Sinaloa' }), // Common practice to refer to the city based on who it was named after // Juarez is in the state of Chihuahua ...([ @@ -420,13 +421,13 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'benito juarez', 'Benito Juárez', 'CD. JUAREZ', - ].includes(record.regionName) && { regionName: 'Chihuahua' }), + ].includes(regionNameTrimmed) && { regionName: 'Chihuahua' }), // San Andres Cholula is in Puebla Mexio which is the state ...([ 'sn andres cholula', 'PUEBLA', 'ZacatlánPue.', - ].includes(record.regionName) && { regionName: 'Puebla' }), + ].includes(regionNameTrimmed) && { regionName: 'Puebla' }), // Mexicalu B.C. Mexico B.C. is Baja California ...([ 'Mexicalu B.C. Mexico', @@ -439,7 +440,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'Baja Calif', 'BC', 'BAJA CALIFORNIA SUR', - ].includes(record.regionName) && { regionName: 'Baja California' }), + ].includes(regionNameTrimmed) && { regionName: 'Baja California' }), ...([ 'Coahuila', 'coahuila', @@ -448,51 +449,51 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'COahuila', 'Torreon', 'Monclova', - ].includes(record.regionName) && { regionName: 'Coahuila de Zaragoza' }), - ...(record.regionName === 'QUINTANA ROO' && { regionName: 'Quintana Roo' }), + ].includes(regionNameTrimmed) && { regionName: 'Coahuila de Zaragoza' }), + ...(regionNameTrimmed === 'QUINTANA ROO' && { regionName: 'Quintana Roo' }), ...([ 'tams', 'tam', 'Tam', 'matamoros', 'TAMPICO', - ].includes(record.regionName) && { regionName: 'Tamaulipas' }), + ].includes(regionNameTrimmed) && { regionName: 'Tamaulipas' }), }), // Occassionally the Canadians don't see to want to put they're from Canada - ...(record.regionName === 'Alberta' && { countryName: 'Canada' }), - ...(record.regionName === 'NL' && { regionName: 'Newfoundland and Labrador', countryName: 'Canada' }), - ...(record.regionName === 'Ontario' && { countryName: 'Canada' }), + ...(regionNameTrimmed === 'Alberta' && { countryName: 'Canada' }), + ...(regionNameTrimmed === 'NL' && { regionName: 'Newfoundland and Labrador', countryName: 'Canada' }), + ...(regionNameTrimmed === 'Ontario' && { countryName: 'Canada' }), // Canada, eh? ...(record.countryName === 'Canada' && { - ...(record.regionName === 'AB' && { regionName: 'Alberta' }), - ...(record.regionName === 'ab' && { regionName: 'Alberta' }), - ...(record.regionName === 'alberta' && { regionName: 'Alberta' }), - ...(record.regionName === 'BC' && { regionName: 'British Columbia' }), - ...(record.regionName === 'bc' && { regionName: 'British Columbia' }), - ...(record.regionName === 'BC - British Columbia' && { regionName: 'British Columbia' }), - ...(record.regionName === 'B.C.' && { regionName: 'British Columbia' }), - ...(record.regionName === 'b.c.' && { regionName: 'British Columbia' }), - ...(record.regionName === 'manitoba' && { regionName: 'Manitoba' }), - ...(record.regionName === 'NS' && { regionName: 'Nova Scotia' }), - ...(record.regionName === 'Qc' && { regionName: 'Quebec' }), - ...(record.regionName === 'qc' && { regionName: 'Quebec' }), - ...(record.regionName === 'quebec' && { regionName: 'Quebec' }), - ...(record.regionName === 'quebec' && { regionName: 'Québec' }), - ...(record.regionName === 'Newfoundland' && { regionName: 'Newfoundland and Labrador' }), - ...(record.regionName === 'NL' && { regionName: 'Newfoundland and Labrador' }), - ...(record.regionName === 'ONTARIO' && { regionName: 'Ontario' }), - ...(record.regionName === 'ontario' && { regionName: 'Ontario' }), - ...(record.regionName === 'Ontario' && { regionName: 'Ontario' }), - ...(record.regionName === 'on' && { regionName: 'Ontario' }), - ...(record.regionName === 'ON' && { regionName: 'Ontario' }), - ...(record.regionName === 'ont' && { regionName: 'Ontario' }), - ...(record.regionName === 'Ont' && { regionName: 'Ontario' }), - ...(record.regionName === 'otario' && { regionName: 'Ontario' }), - ...(record.regionName === 'On' && { regionName: 'Ontario' }), - ...(record.regionName === 'Yukon Territory', { regionName: 'Yukon' }), - ...(record.regionName === 'sk', { regionName: 'Saskatchewan' }), - ...(record.regionName === 'New-Brunswick' && { regionName: 'New Brunswick' }), + ...(regionNameTrimmed === 'AB' && { regionName: 'Alberta' }), + ...(regionNameTrimmed === 'ab' && { regionName: 'Alberta' }), + ...(regionNameTrimmed === 'alberta' && { regionName: 'Alberta' }), + ...(regionNameTrimmed === 'BC' && { regionName: 'British Columbia' }), + ...(regionNameTrimmed === 'bc' && { regionName: 'British Columbia' }), + ...(regionNameTrimmed === 'BC - British Columbia' && { regionName: 'British Columbia' }), + ...(regionNameTrimmed === 'B.C.' && { regionName: 'British Columbia' }), + ...(regionNameTrimmed === 'b.c.' && { regionName: 'British Columbia' }), + ...(regionNameTrimmed === 'manitoba' && { regionName: 'Manitoba' }), + ...(regionNameTrimmed === 'NS' && { regionName: 'Nova Scotia' }), + ...(regionNameTrimmed === 'Qc' && { regionName: 'Quebec' }), + ...(regionNameTrimmed === 'qc' && { regionName: 'Quebec' }), + ...(regionNameTrimmed === 'quebec' && { regionName: 'Quebec' }), + ...(regionNameTrimmed === 'quebec' && { regionName: 'Québec' }), + ...(regionNameTrimmed === 'Newfoundland' && { regionName: 'Newfoundland and Labrador' }), + ...(regionNameTrimmed === 'NL' && { regionName: 'Newfoundland and Labrador' }), + ...(regionNameTrimmed === 'ONTARIO' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'ontario' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'Ontario' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'on' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'ON' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'ont' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'Ont' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'otario' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'On' && { regionName: 'Ontario' }), + ...(regionNameTrimmed === 'Yukon Territory', { regionName: 'Yukon' }), + ...(regionNameTrimmed === 'sk', { regionName: 'Saskatchewan' }), + ...(regionNameTrimmed === 'New-Brunswick' && { regionName: 'New Brunswick' }), } ), @@ -585,7 +586,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'i', 'no', 'T\'ai-wan', - ].includes(record.regionName) && { + ].includes(regionNameTrimmed) && { regionName: undefined, }), }), {}); From a0ce942b6653d3e81d1425e89f88777a20e6fbcc Mon Sep 17 00:00:00 2001 From: Jake Collins Date: Fri, 7 Apr 2023 09:44:50 -0500 Subject: [PATCH 22/24] Strip trailing periods off normalized email --- services/import/src/validate.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index d5d75a4a..ae40cb7a 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -596,7 +596,8 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe if (!filtered._id) throw new Error('Missing `_id` column, verify CSV!'); const { _id } = filtered; if (!isInternal) delete filtered._id; - const email = normalizeEmail(filtered.email); + // Strip trailing periods if applicable + const email = normalizeEmail(filtered.email).replace(/\.+$/, ''); const [, domain] = email.split('@'); const externalId = Buffer.from(_id).toString('base64'); const normalized = { @@ -613,7 +614,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe namespace: { provider: 'backoffice', tenant: 'smg', type: 'member' }, }, }), - email: normalizeEmail(filtered.email), + email, domain, verified: false, receiveEmail: filtered.receiveEmail === 'TRUE', From d58f1c8eef0050c35ef48f8e9c5be0a4b4446503 Mon Sep 17 00:00:00 2001 From: Josh Worden Date: Tue, 12 Sep 2023 11:26:44 -0500 Subject: [PATCH 23/24] Add additional region mappings --- services/import/src/validate.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index ae40cb7a..a5f2b5df 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -341,6 +341,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'M�xico D.F.', 'D.F', 'Federal District of Mexico', + 'Colonia Doctores', ].includes(regionNameTrimmed) && { regionName: 'Ciudad de México' }), ...([ 'Michoacan', @@ -382,12 +383,15 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'guadalajara', 'ZAPOPAN', 'Guadalajara', + 'Ouest', ].includes(regionNameTrimmed) && { regionName: 'Jalisco' }), ...([ 'SONORA', 'HERMOSILLO', 'SON', 'Hermosillo', + 'Ciudad Obregón', + 'OBREGON', ].includes(regionNameTrimmed) && { regionName: 'Sonora' }), ...(regionNameTrimmed === 'chihuahua' && { regionName: 'Chihuahua' }), ...([ From 543ee0aa3d4425338e4133e94fd8f8562aaafa41 Mon Sep 17 00:00:00 2001 From: Shinsina Date: Wed, 11 Oct 2023 13:42:30 -0500 Subject: [PATCH 24/24] Validation exceptions for Aunt Minnie Europe --- services/import/src/validate.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/services/import/src/validate.js b/services/import/src/validate.js index a5f2b5df..6b7961bb 100644 --- a/services/import/src/validate.js +++ b/services/import/src/validate.js @@ -246,6 +246,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe ...(record.countryName === 'Hong Kong SAR China' && { countryName: 'Hong Kong' }), ...(record.countryName === 'Falkland Islands' && { countryName: 'Falkland Islands (Malvinas)' }), ...(record.countryName === 'Western Samoa' && { countryName: 'Samoa' }), + ...(record.countryName === 'Republic of San Marino' && { countryName: 'San Marino' }), // Regions @@ -444,6 +445,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'Baja Calif', 'BC', 'BAJA CALIFORNIA SUR', + 'bc', ].includes(regionNameTrimmed) && { regionName: 'Baja California' }), ...([ 'Coahuila', @@ -521,6 +523,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'French Antilles', // Someone's name is in the wrong field 'Diego Garcia', + '87-100', ].includes(record.countryName) && { countryName: undefined, }), @@ -590,6 +593,7 @@ module.exports = async (records = [], applicationId, limit = 10, errorOnBadAnswe 'i', 'no', 'T\'ai-wan', + 'I am not in the U.S. or CANADA', ].includes(regionNameTrimmed) && { regionName: undefined, }),