From a5044f1bffe5ccb76c4d94cc708c86921fce3ecc Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Thu, 8 Dec 2022 16:28:47 +0300 Subject: [PATCH 01/37] feat: initiate BigQuery DLP extension --- bigquery-dlp-function/POSTINSTALL.md | 0 bigquery-dlp-function/PREINSTALL.md | 0 bigquery-dlp-function/README.md | 0 bigquery-dlp-function/extension.yaml | 103 ++++++++++++++ bigquery-dlp-function/functions/lib/config.js | 22 +++ .../functions/lib/config.js.map | 1 + bigquery-dlp-function/functions/lib/index.js | 93 +++++++++++++ .../functions/lib/index.js.map | 1 + bigquery-dlp-function/functions/package.json | 31 +++++ bigquery-dlp-function/functions/src/config.ts | 20 +++ bigquery-dlp-function/functions/src/index.ts | 131 ++++++++++++++++++ .../functions/src/types/call_mode.ts | 6 + bigquery-dlp-function/functions/tsconfig.json | 15 ++ firestore-auth-claims/functions/tsconfig.json | 12 +- 14 files changed, 429 insertions(+), 6 deletions(-) create mode 100644 bigquery-dlp-function/POSTINSTALL.md create mode 100644 bigquery-dlp-function/PREINSTALL.md create mode 100644 bigquery-dlp-function/README.md create mode 100644 bigquery-dlp-function/extension.yaml create mode 100644 bigquery-dlp-function/functions/lib/config.js create mode 100644 bigquery-dlp-function/functions/lib/config.js.map create mode 100644 bigquery-dlp-function/functions/lib/index.js create mode 100644 bigquery-dlp-function/functions/lib/index.js.map create mode 100644 bigquery-dlp-function/functions/package.json create mode 100644 bigquery-dlp-function/functions/src/config.ts create mode 100644 bigquery-dlp-function/functions/src/index.ts create mode 100644 bigquery-dlp-function/functions/src/types/call_mode.ts create mode 100644 bigquery-dlp-function/functions/tsconfig.json diff --git a/bigquery-dlp-function/POSTINSTALL.md b/bigquery-dlp-function/POSTINSTALL.md new file mode 100644 index 00000000..e69de29b diff --git a/bigquery-dlp-function/PREINSTALL.md b/bigquery-dlp-function/PREINSTALL.md new file mode 100644 index 00000000..e69de29b diff --git a/bigquery-dlp-function/README.md b/bigquery-dlp-function/README.md new file mode 100644 index 00000000..e69de29b diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml new file mode 100644 index 00000000..fd36ec42 --- /dev/null +++ b/bigquery-dlp-function/extension.yaml @@ -0,0 +1,103 @@ +# Learn detailed information about the fields of an extension.yaml file in the docs: +# https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml + +name: bigquery-dlp-function # Identifier for your extension +version: 0.0.1 # Follow semver versioning +specVersion: v1beta # Version of the Firebase Extensions specification + +author: + authorName: Firebase + url: https://firebase.google.com + +# Friendly display name for your extension (~3-5 words) +displayName: BigQuery DLP Remote Function + +# Brief description of the task your extension performs (~1 sentence) +description: TODO + +license: Apache-2.0 # https://spdx.org/licenses/ + +# Public URL for the source code of your extension +sourceUrl: TODO + +# Specify whether a paid-tier billing plan is required to use your extension. +# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#billing-required-field +billingRequired: true + +# In an `apis` field, list any Google APIs (like Cloud Translation, BigQuery, etc.) +# required for your extension to operate. +# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#apis-field +apis: + - apiName: bigquery.googleapis.com + reason: Powers all BigQuery tasks performed by the extension. + +# In a `roles` field, list any IAM access roles required for your extension to operate. +# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#roles-field +roles: + - role: actions.Viewer + reason: Allows the extension to create BigQuery jobs. + +# In the `resources` field, list each of your extension's functions, including the trigger for each function. +# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#resources-field +resources: + - name: deidentifyData + type: firebaseextensions.v1beta.function + description: TODO + properties: + location: ${param:LOCATION} + runtime: nodejs14 + httpsTrigger: {} + +# In the `params` field, set up your extension's user-configured parameters. +# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#params-field +params: + - param: LOCATION + label: Cloud Functions location + description: >- + Where do you want to deploy the functions created for this extension? You + usually want a location close to your database. For help selecting a + location, refer to the [location selection + guide](https://firebase.google.com/docs/functions/locations). + type: select + options: + - label: Iowa (us-central1) + value: us-central1 + - label: South Carolina (us-east1) + value: us-east1 + - label: Northern Virginia (us-east4) + value: us-east4 + - label: Los Angeles (us-west2) + value: us-west2 + - label: Salt Lake City (us-west3) + value: us-west3 + - label: Las Vegas (us-west4) + value: us-west4 + - label: Belgium (europe-west1) + value: europe-west1 + - label: London (europe-west2) + value: europe-west2 + - label: Frankfurt (europe-west3) + value: europe-west3 + - label: Zurich (europe-west6) + value: europe-west6 + - label: Hong Kong (asia-east2) + value: asia-east2 + - label: Tokyo (asia-northeast1) + value: asia-northeast1 + - label: Osaka (asia-northeast2) + value: asia-northeast2 + - label: Seoul (asia-northeast3) + value: asia-northeast3 + - label: Mumbai (asia-south1) + value: asia-south1 + - label: Jakarta (asia-southeast2) + value: asia-southeast2 + - label: Montreal (northamerica-northeast1) + value: northamerica-northeast1 + - label: Sao Paulo (southamerica-east1) + value: southamerica-east1 + - label: Sydney (australia-southeast1) + value: australia-southeast1 + default: us-central1 + required: true + immutable: true diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js new file mode 100644 index 00000000..444f53dd --- /dev/null +++ b/bigquery-dlp-function/functions/lib/config.js @@ -0,0 +1,22 @@ +"use strict"; +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +Object.defineProperty(exports, "__esModule", { value: true }); +exports.default = { + projectId: process.env.PROJECT_ID, + servicePath: "../extensions-sample-firebase-adminsdk.json", +}; +//# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map new file mode 100644 index 00000000..5b5dbe2d --- /dev/null +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -0,0 +1 @@ +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,WAAW,EAAE,6CAA6C;CAC3D,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js new file mode 100644 index 00000000..1a8df030 --- /dev/null +++ b/bigquery-dlp-function/functions/lib/index.js @@ -0,0 +1,93 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.deidentifyData = void 0; +const functions = require("firebase-functions"); +const dlp_1 = require("@google-cloud/dlp"); +const bigquery_connection_1 = require("@google-cloud/bigquery-connection"); +const config_1 = require("./config"); +const CALL_MODE_KEY = "mode"; +const TRANSFORM_ALGO_KEY = "algo"; +// Instantiates a client +const dlp = new dlp_1.default.DlpServiceClient({ + keyFilename: config_1.default.servicePath, +}); +const bigqueryClient = new bigquery_connection_1.ConnectionServiceClient({ + keyFilename: config_1.default.servicePath, +}); +const parent = `projects/${config_1.default.projectId}/locations/global`; +const deidentifyConfig = { + parent: parent, + deidentifyConfig: { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: "x", + numberToMask: 5, + }, + }, + }, + ], + }, + }, +}; +/** + * Deidentify sensitive data in a string using the Data Loss Prevention API. + * + * @param {string} text The text to deidentify. + * + * @returns {string} The deidentified text. + */ +async function deidentifyWithMask(rows) { + var _a; + const deidentifiedItems = []; + for (const row of rows) { + const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: row.value } }); + const [response] = await dlp.deidentifyContent(request); + deidentifiedItems.push((_a = response.item) === null || _a === void 0 ? void 0 : _a.value); + console.log(response); + } + return deidentifiedItems; +} +exports.deidentifyData = functions.https.onCall(async (data, ctx) => { + bigqueryClient.createConnection({ + parent: parent, + connection: { name: "test", cloudResource: {} }, + connectionId: "test", + }); + const bqrequest = data; + const options = checkNotNull(bqrequest.userDefinedContext); + const callMode = identifyCallMode(options); + switch (callMode) { + case CallMode.DEIDENTIFY: + break; + case CallMode.REIDENTIFY: + break; + } + const items = bqrequest.calls; + const result = await deidentifyWithMask(items); + return JSON.stringify(result); +}); +function checkNotNull(options) { + if (options == null) { + throw new Error("userDefinedContext is required. Found null."); + } + return { replies: options }; +} +function identifyCallMode(options) { + var callMode = CALL_MODE_KEY in options ? options[CALL_MODE_KEY] : null; + if (CALL_MODE_KEY in options) { + const callModeEnum = callMode; + return callModeEnum; + } + else { + return CallMode.DEIDENTIFY; + } +} +var CallMode; +(function (CallMode) { + CallMode["DEIDENTIFY"] = "DEIDENTIFY"; + CallMode["REIDENTIFY"] = "REIDENTIFY"; +})(CallMode || (CallMode = {})); +//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map new file mode 100644 index 00000000..42c205d5 --- /dev/null +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -0,0 +1 @@ +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,2CAAoC;AACpC,2EAA4E;AAC5E,qCAA8B;AAE9B,MAAM,aAAa,GAAG,MAAM,CAAC;AAC7B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,wBAAwB;AACxB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,CAAC;IACnC,WAAW,EAAE,gBAAM,CAAC,WAAW;CAChC,CAAC,CAAC;AACH,MAAM,cAAc,GAAG,IAAI,6CAAuB,CAAC;IACjD,WAAW,EAAE,gBAAM,CAAC,WAAW;CAChC,CAAC,CAAC;AAMH,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,mBAAmB,CAAC;AAU/D,MAAM,gBAAgB,GAAG;IACvB,MAAM,EAAE,MAAM;IACd,gBAAgB,EAAE;QAChB,uBAAuB,EAAE;YACvB,eAAe,EAAE;gBACf;oBACE,uBAAuB,EAAE;wBACvB,mBAAmB,EAAE;4BACnB,gBAAgB,EAAE,GAAG;4BACrB,YAAY,EAAE,CAAC;yBAChB;qBACF;iBACF;aACF;SACF;KACF;CACF,CAAC;AAEF;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAY;;IAC5C,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,GAC3B,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;QAE7C,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACvB;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IACvE,cAAc,CAAC,gBAAgB,CAAC;QAC9B,MAAM,EAAE,MAAM;QACd,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE;QAC/C,YAAY,EAAE,MAAM;KACrB,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAiB,CAAC;IAEpC,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,QAAQ,QAAQ,EAAE;QAChB,KAAK,QAAQ,CAAC,UAAU;YACtB,MAAM;QAER,KAAK,QAAQ,CAAC,UAAU;YACtB,MAAM;KACT;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,QAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,IAAK,QAGJ;AAHD,WAAK,QAAQ;IACX,qCAAyB,CAAA;IACzB,qCAAyB,CAAA;AAC3B,CAAC,EAHI,QAAQ,KAAR,QAAQ,QAGZ"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/package.json b/bigquery-dlp-function/functions/package.json new file mode 100644 index 00000000..0e0e8c45 --- /dev/null +++ b/bigquery-dlp-function/functions/package.json @@ -0,0 +1,31 @@ +{ + "name": "functions", + "scripts": { + "lint": "eslint --ext .js,.ts .", + "build": "tsc", + "build:watch": "tsc --watch", + "serve": "npm run build && firebase emulators:start --only functions", + "shell": "npm run build && firebase functions:shell", + "start": "npm run shell", + "deploy": "firebase deploy --only functions", + "logs": "firebase functions:log" + }, + "main": "lib/index.js", + "dependencies": { + "@google-cloud/bigquery": "^6.0.3", + "@google-cloud/bigquery-connection": "^2.0.2", + "@google-cloud/dlp": "^4.2.0", + "firebase-admin": "^10.0.2", + "firebase-functions": "^3.18.0" + }, + "devDependencies": { + "@typescript-eslint/eslint-plugin": "^5.12.0", + "@typescript-eslint/parser": "^5.12.0", + "eslint": "^8.9.0", + "eslint-config-google": "^0.14.0", + "eslint-plugin-import": "^2.25.4", + "firebase-functions-test": "^0.2.0", + "typescript": "^4.5.4" + }, + "private": true +} diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts new file mode 100644 index 00000000..9d67f322 --- /dev/null +++ b/bigquery-dlp-function/functions/src/config.ts @@ -0,0 +1,20 @@ +/* + * Copyright 2019 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +export default { + projectId: process.env.PROJECT_ID, + servicePath: "../extensions-sample-firebase-adminsdk.json", +}; diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts new file mode 100644 index 00000000..d4480bbf --- /dev/null +++ b/bigquery-dlp-function/functions/src/index.ts @@ -0,0 +1,131 @@ +import * as functions from "firebase-functions"; +import DLP from "@google-cloud/dlp"; +import { ConnectionServiceClient } from "@google-cloud/bigquery-connection"; +import config from "./config"; +import CallMode from "./types/call_mode"; + +const CALL_MODE_KEY = "mode"; +const TRANSFORM_ALGO_KEY = "algo"; + +// Instantiates a client +const dlp = new DLP.DlpServiceClient({ + keyFilename: config.servicePath, +}); +const bigqueryClient = new ConnectionServiceClient({ + keyFilename: config.servicePath, +}); + +interface Item { + value: string; +} + +interface BQRequest { + requestId: string; + callecallerrIp: string; + sessionUser: string; + userDefinedContext: {}; + calls: []; +} + +const deidentifyConfig = { + parent: parent, + deidentifyConfig: { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: "x", + numberToMask: 5, + }, + }, + }, + ], + }, + }, +}; + +/** + * Deidentify sensitive data in a string using the Data Loss Prevention API. + * + * @param {string} text The text to deidentify. + * + * @returns {string} The deidentified text. + */ +async function deidentifyWithMask(rows: Item[]) { + const deidentifiedItems = []; + + const parent = `projects/${config.projectId}/locations/${location}`; + + for (const row of rows) { + const request = { + ...deidentifyConfig, + item: { value: row.value }, + parent: parent, + }; + + const [response] = await dlp.deidentifyContent(request); + deidentifiedItems.push(response.item?.value); + + console.log(response); + } + + return deidentifiedItems; +} + +export const deidentifyData = functions.https.onCall(async (data, ctx) => { + const bqrequest = data as BQRequest; + + const options = checkNotNull(bqrequest.userDefinedContext); + const callMode = identifyCallMode(options); + + switch (callMode) { + case CallMode.DEIDENTIFY: + break; + case CallMode.REIDENTIFY: + break; + } + + const items = bqrequest.calls; + const result = await deidentifyWithMask(items); + + return JSON.stringify(result); +}); + +function checkNotNull(options: BQRequest["userDefinedContext"]) { + if (options == null) { + throw new Error("userDefinedContext is required. Found null."); + } + + return { replies: options }; +} + +function identifyCallMode(options: any): string { + var callMode = + CALL_MODE_KEY in options ? (options[CALL_MODE_KEY] as string) : null; + + if (CALL_MODE_KEY in options) { + const callModeEnum = callMode as CallMode; + + return callModeEnum; + } else { + return CallMode.DEIDENTIFY; + } +} + +exports.onInstall = functions.tasks.taskQueue().onDispatch(async (task) => { + const data = task.data; + console.log("Task received: ", data); + + const projectId = data.projectId; + const location = data.location; + const parent = `projects/${projectId}/locations/${location}`; + + const connection = bigqueryClient.createConnection({ + parent: parent, + connection: { name: "deidentifyData", cloudResource: {} }, + connectionId: "connectionId", + }); + + console.log("Connection: ", connection); +}); diff --git a/bigquery-dlp-function/functions/src/types/call_mode.ts b/bigquery-dlp-function/functions/src/types/call_mode.ts new file mode 100644 index 00000000..37041655 --- /dev/null +++ b/bigquery-dlp-function/functions/src/types/call_mode.ts @@ -0,0 +1,6 @@ +enum CallMode { + DEIDENTIFY = "DEIDENTIFY", + REIDENTIFY = "REIDENTIFY", +} + +export default CallMode; diff --git a/bigquery-dlp-function/functions/tsconfig.json b/bigquery-dlp-function/functions/tsconfig.json new file mode 100644 index 00000000..7ce05d03 --- /dev/null +++ b/bigquery-dlp-function/functions/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "module": "commonjs", + "noImplicitReturns": true, + "noUnusedLocals": true, + "outDir": "lib", + "sourceMap": true, + "strict": true, + "target": "es2017" + }, + "compileOnSave": true, + "include": [ + "src" + ] +} diff --git a/firestore-auth-claims/functions/tsconfig.json b/firestore-auth-claims/functions/tsconfig.json index a9ed863a..9d2d8f96 100644 --- a/firestore-auth-claims/functions/tsconfig.json +++ b/firestore-auth-claims/functions/tsconfig.json @@ -1,13 +1,13 @@ { "compilerOptions": { + "lib": ["esnext.asynciterable", "es2020", "es6"], + "outDir": "lib", "module": "commonjs", "noImplicitReturns": true, - "noUnusedLocals": true, - "outDir": "lib", - "sourceMap": true, - "strict": true, - "target": "es2017" + "sourceMap": false, + "target": "es6" }, "compileOnSave": true, - "include": ["src"] + "include": ["src"], + "exclude": ["node_modules"] } From eb695e06f5250b4f0198388dd23b804b744ae35e Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Thu, 8 Dec 2022 16:30:59 +0300 Subject: [PATCH 02/37] add path to emulator --- _emulator/firebase.json | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/_emulator/firebase.json b/_emulator/firebase.json index 0f5728b6..43b8a726 100644 --- a/_emulator/firebase.json +++ b/_emulator/firebase.json @@ -1,7 +1,8 @@ { "extensions": { "firestore-record-user-acknowledgements": "../firestore-record-user-acknowledgements", - "firestore-bundle-server": "../firestore-bundle-server" + "firestore-bundle-server": "../firestore-bundle-server", + "bigquery-dlp-function": "../bigquery-dlp-function" }, "storage": { "rules": "storage.rules" @@ -35,7 +36,11 @@ }, "hosting": { "public": "dist", - "ignore": ["firebase.json", "**/.*", "**/node_modules/**"], + "ignore": [ + "firebase.json", + "**/.*", + "**/node_modules/**" + ], "rewrites": [ { "source": "/bundles/*", From 64ae97d23170fbe80fd02bd25cf8d2219761a922 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 9 Dec 2022 22:22:06 +0300 Subject: [PATCH 03/37] lifecyle event --- bigquery-dlp-function/extension.yaml | 41 ++-- bigquery-dlp-function/functions/lib/config.js | 3 +- .../functions/lib/config.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 159 +++++++++----- .../functions/lib/index.js.map | 2 +- .../functions/lib/types/call_mode.js | 9 + .../functions/lib/types/call_mode.js.map | 1 + bigquery-dlp-function/functions/package.json | 2 +- bigquery-dlp-function/functions/src/config.ts | 3 +- bigquery-dlp-function/functions/src/index.ts | 205 ++++++++++++------ 10 files changed, 283 insertions(+), 144 deletions(-) create mode 100644 bigquery-dlp-function/functions/lib/types/call_mode.js create mode 100644 bigquery-dlp-function/functions/lib/types/call_mode.js.map diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index fd36ec42..66dc3a5a 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -9,37 +9,36 @@ author: authorName: Firebase url: https://firebase.google.com -# Friendly display name for your extension (~3-5 words) displayName: BigQuery DLP Remote Function -# Brief description of the task your extension performs (~1 sentence) description: TODO license: Apache-2.0 # https://spdx.org/licenses/ -# Public URL for the source code of your extension sourceUrl: TODO -# Specify whether a paid-tier billing plan is required to use your extension. -# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#billing-required-field billingRequired: true -# In an `apis` field, list any Google APIs (like Cloud Translation, BigQuery, etc.) -# required for your extension to operate. -# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#apis-field apis: - apiName: bigquery.googleapis.com reason: Powers all BigQuery tasks performed by the extension. + - apiName: bigqueryconnection.googleapis.com + reason: Allows the extension to create a BigQuery connection. -# In a `roles` field, list any IAM access roles required for your extension to operate. -# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#roles-field roles: - - role: actions.Viewer - reason: Allows the extension to create BigQuery jobs. + # - role: actions.Viewer + # reason: Allows the extension to use services. + - role: bigquery.connectionAdmin + reason: Allows the extension to create a BigQuery connection. -# In the `resources` field, list each of your extension's functions, including the trigger for each function. -# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#resources-field resources: + - name: createBigQueryConnection + type: firebaseextensions.v1beta.function + description: Creates a BigQuery connection. + properties: + location: ${param:LOCATION} + runtime: nodejs14 + taskQueueTrigger: {} - name: deidentifyData type: firebaseextensions.v1beta.function description: TODO @@ -48,9 +47,14 @@ resources: runtime: nodejs14 httpsTrigger: {} -# In the `params` field, set up your extension's user-configured parameters. -# Learn more in the docs: https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml#params-field params: + - param: DATASET_ID + label: BigQuery Dataset ID + description: >- + The ID of the dataset where the extension will create a connection. + type: string + default: deidentified_data + required: true - param: LOCATION label: Cloud Functions location description: >- @@ -101,3 +105,8 @@ params: default: us-central1 required: true immutable: true + +lifecycleEvents: + onInstall: + function: createBigQueryConnection + processingMessage: "Create BigQuery connections" diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js index 444f53dd..3b962d0a 100644 --- a/bigquery-dlp-function/functions/lib/config.js +++ b/bigquery-dlp-function/functions/lib/config.js @@ -17,6 +17,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.default = { projectId: process.env.PROJECT_ID, - servicePath: "../extensions-sample-firebase-adminsdk.json", + datasetId: process.env.DATASET_ID, + location: process.env.LOCATION, }; //# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map index 5b5dbe2d..69b88d26 100644 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -1 +1 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,WAAW,EAAE,6CAA6C;CAC3D,CAAC"} \ No newline at end of file +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;CAC/B,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 1a8df030..c3a30b88 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -2,36 +2,18 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.deidentifyData = void 0; const functions = require("firebase-functions"); +const admin = require("firebase-admin"); const dlp_1 = require("@google-cloud/dlp"); const bigquery_connection_1 = require("@google-cloud/bigquery-connection"); +const bigquery_1 = require("@google-cloud/bigquery"); +const extensions_1 = require("firebase-admin/extensions"); const config_1 = require("./config"); +const call_mode_1 = require("./types/call_mode"); const CALL_MODE_KEY = "mode"; -const TRANSFORM_ALGO_KEY = "algo"; -// Instantiates a client -const dlp = new dlp_1.default.DlpServiceClient({ - keyFilename: config_1.default.servicePath, -}); -const bigqueryClient = new bigquery_connection_1.ConnectionServiceClient({ - keyFilename: config_1.default.servicePath, -}); -const parent = `projects/${config_1.default.projectId}/locations/global`; -const deidentifyConfig = { - parent: parent, - deidentifyConfig: { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: "x", - numberToMask: 5, - }, - }, - }, - ], - }, - }, -}; +admin.initializeApp(); +const dlp = new dlp_1.default.DlpServiceClient(); +const bigqueryClient = new bigquery_1.BigQuery(); +const bigqueryConnectionClient = new bigquery_connection_1.ConnectionServiceClient(); /** * Deidentify sensitive data in a string using the Data Loss Prevention API. * @@ -42,32 +24,52 @@ const deidentifyConfig = { async function deidentifyWithMask(rows) { var _a; const deidentifiedItems = []; + const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + const deidentifyConfig = { + parent: parent, + deidentifyConfig: { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: "x", + numberToMask: 5, + }, + }, + }, + ], + }, + }, + }; for (const row of rows) { - const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: row.value } }); - const [response] = await dlp.deidentifyContent(request); - deidentifiedItems.push((_a = response.item) === null || _a === void 0 ? void 0 : _a.value); - console.log(response); + for (const value in row) { + const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: value }, parent: parent }); + const [response] = await dlp.deidentifyContent(request); + deidentifiedItems.push((_a = response.item) === null || _a === void 0 ? void 0 : _a.value); + functions.logger.debug(response); + } } return deidentifiedItems; } -exports.deidentifyData = functions.https.onCall(async (data, ctx) => { - bigqueryClient.createConnection({ - parent: parent, - connection: { name: "test", cloudResource: {} }, - connectionId: "test", - }); - const bqrequest = data; - const options = checkNotNull(bqrequest.userDefinedContext); - const callMode = identifyCallMode(options); - switch (callMode) { - case CallMode.DEIDENTIFY: - break; - case CallMode.REIDENTIFY: - break; +exports.deidentifyData = functions.https.onRequest(async (request, response) => { + const { calls } = request.body; + functions.logger.debug("Incoming request from BigQuery", request.body); + // const options = checkNotNull(userDefinedContext); + // const callMode = identifyCallMode(options); + // switch (callMode) { + // case CallMode.DEIDENTIFY: + // break; + // case CallMode.REIDENTIFY: + // break; + // } + try { + const result = await deidentifyWithMask(calls); + response.send(JSON.stringify({ replies: result })); + } + catch (error) { + response.status(500).send(`Something wrong happend ${error}`); } - const items = bqrequest.calls; - const result = await deidentifyWithMask(items); - return JSON.stringify(result); }); function checkNotNull(options) { if (options == null) { @@ -82,12 +84,63 @@ function identifyCallMode(options) { return callModeEnum; } else { - return CallMode.DEIDENTIFY; + return call_mode_1.default.DEIDENTIFY; } } -var CallMode; -(function (CallMode) { - CallMode["DEIDENTIFY"] = "DEIDENTIFY"; - CallMode["REIDENTIFY"] = "REIDENTIFY"; -})(CallMode || (CallMode = {})); +exports.createBigQueryConnection = functions.tasks + .taskQueue() + .onDispatch(async (task) => { + const runtime = (0, extensions_1.getExtensions)().runtime(); + console.log("Task received => ", task); + const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + functions.logger.debug(`${parent}/connections/deidentify`); + try { + const connection1 = await bigqueryConnectionClient.createConnection({ + parent: parent, + connectionId: `${parent}/connections/deidentify`, + connection: { + cloudResource: { + serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", + }, + name: "deidentify", + friendlyName: "Deidentify Data", + }, + }); + const connection2 = await bigqueryConnectionClient.createConnection({ + parent: parent, + connectionId: `projects/${config_1.default.projectId}/locations/${config_1.default.location}/connections/reidentify`, + connection: { + cloudResource: { + serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", + }, + name: "reidentify", + friendlyName: "Reidentify Data", + }, + }); + functions.logger.info("Connection 1 => ", connection1); + functions.logger.info("Connection 2 => ", connection2); + if (connection1 && connection2) { + const query = `CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deindetify(data STRING) RETURNS STRING + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.ext-bq-dlp.deidentify\` + OPTIONS ( + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + )`; + const options = { + query: query, + location: "US", + }; + // Run the query as a job + const [job] = await bigqueryClient.createQueryJob(options); + functions.logger.debug(`Job ${job.id} started.`); + // Wait for the query to finish + const [rows] = await job.getQueryResults(); + functions.logger.debug("Rows: ", rows); + await runtime.setProcessingState("PROCESSING_COMPLETE", "Connections created successfully."); + } + } + catch (error) { + functions.logger.error(error); + await runtime.setProcessingState("PROCESSING_FAILED", "Connections were not created."); + } +}); //# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 42c205d5..9cc4f5ef 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,2CAAoC;AACpC,2EAA4E;AAC5E,qCAA8B;AAE9B,MAAM,aAAa,GAAG,MAAM,CAAC;AAC7B,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAElC,wBAAwB;AACxB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,CAAC;IACnC,WAAW,EAAE,gBAAM,CAAC,WAAW;CAChC,CAAC,CAAC;AACH,MAAM,cAAc,GAAG,IAAI,6CAAuB,CAAC;IACjD,WAAW,EAAE,gBAAM,CAAC,WAAW;CAChC,CAAC,CAAC;AAMH,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,mBAAmB,CAAC;AAU/D,MAAM,gBAAgB,GAAG;IACvB,MAAM,EAAE,MAAM;IACd,gBAAgB,EAAE;QAChB,uBAAuB,EAAE;YACvB,eAAe,EAAE;gBACf;oBACE,uBAAuB,EAAE;wBACvB,mBAAmB,EAAE;4BACnB,gBAAgB,EAAE,GAAG;4BACrB,YAAY,EAAE,CAAC;yBAChB;qBACF;iBACF;aACF;SACF;KACF;CACF,CAAC;AAEF;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAY;;IAC5C,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,GAAG,CAAC,KAAK,EAAE,GAC3B,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;QAE7C,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;KACvB;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,GAAG,EAAE,EAAE;IACvE,cAAc,CAAC,gBAAgB,CAAC;QAC9B,MAAM,EAAE,MAAM;QACd,UAAU,EAAE,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,EAAE,EAAE;QAC/C,YAAY,EAAE,MAAM;KACrB,CAAC,CAAC;IACH,MAAM,SAAS,GAAG,IAAiB,CAAC;IAEpC,MAAM,OAAO,GAAG,YAAY,CAAC,SAAS,CAAC,kBAAkB,CAAC,CAAC;IAC3D,MAAM,QAAQ,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC;IAE3C,QAAQ,QAAQ,EAAE;QAChB,KAAK,QAAQ,CAAC,UAAU;YACtB,MAAM;QAER,KAAK,QAAQ,CAAC,UAAU;YACtB,MAAM;KACT;IAED,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC;IAC9B,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/C,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;AAChC,CAAC,CAAC,CAAC;AAEH,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,QAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,IAAK,QAGJ;AAHD,WAAK,QAAQ;IACX,qCAAyB,CAAA;IACzB,qCAAyB,CAAA;AAC3B,CAAC,EAHI,QAAQ,KAAR,QAAQ,QAGZ"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,iDAAyC;AAEzC,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAY/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,oDAAoD;IACpD,8CAA8C;IAE9C,sBAAsB;IACtB,8BAA8B;IAC9B,aAAa;IACb,8BAA8B;IAC9B,aAAa;IACb,IAAI;IAEJ,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;KACpD;IAAC,OAAO,KAAK,EAAE;QACd,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;KAC/D;AACH,CAAC,CACF,CAAC;AAEF,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,mBAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,yBAAyB,CAAC,CAAC;IAE3D,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,MAAM,yBAAyB;YAChD,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,yBAAyB;YAChG,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACvD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAEvD,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG,qBAAqB,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;iCAC9C,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ;;8BAEtC,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;QACzD,CAAC;YAED,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,IAAI;aACf,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/types/call_mode.js b/bigquery-dlp-function/functions/lib/types/call_mode.js new file mode 100644 index 00000000..2a20c9ac --- /dev/null +++ b/bigquery-dlp-function/functions/lib/types/call_mode.js @@ -0,0 +1,9 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +var CallMode; +(function (CallMode) { + CallMode["DEIDENTIFY"] = "DEIDENTIFY"; + CallMode["REIDENTIFY"] = "REIDENTIFY"; +})(CallMode || (CallMode = {})); +exports.default = CallMode; +//# sourceMappingURL=call_mode.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/types/call_mode.js.map b/bigquery-dlp-function/functions/lib/types/call_mode.js.map new file mode 100644 index 00000000..3f8263e5 --- /dev/null +++ b/bigquery-dlp-function/functions/lib/types/call_mode.js.map @@ -0,0 +1 @@ +{"version":3,"file":"call_mode.js","sourceRoot":"","sources":["../../src/types/call_mode.ts"],"names":[],"mappings":";;AAAA,IAAK,QAGJ;AAHD,WAAK,QAAQ;IACX,qCAAyB,CAAA;IACzB,qCAAyB,CAAA;AAC3B,CAAC,EAHI,QAAQ,KAAR,QAAQ,QAGZ;AAED,kBAAe,QAAQ,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/package.json b/bigquery-dlp-function/functions/package.json index 0e0e8c45..b2597a10 100644 --- a/bigquery-dlp-function/functions/package.json +++ b/bigquery-dlp-function/functions/package.json @@ -15,7 +15,7 @@ "@google-cloud/bigquery": "^6.0.3", "@google-cloud/bigquery-connection": "^2.0.2", "@google-cloud/dlp": "^4.2.0", - "firebase-admin": "^10.0.2", + "firebase-admin": "^11.3.0", "firebase-functions": "^3.18.0" }, "devDependencies": { diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 9d67f322..4f63d0c0 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -16,5 +16,6 @@ export default { projectId: process.env.PROJECT_ID, - servicePath: "../extensions-sample-firebase-adminsdk.json", + datasetId: process.env.DATASET_ID, + location: process.env.LOCATION, }; diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index d4480bbf..723a30aa 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -1,23 +1,22 @@ import * as functions from "firebase-functions"; +import * as admin from "firebase-admin"; import DLP from "@google-cloud/dlp"; import { ConnectionServiceClient } from "@google-cloud/bigquery-connection"; +import { BigQuery } from "@google-cloud/bigquery"; +import { getExtensions } from "firebase-admin/extensions"; + import config from "./config"; import CallMode from "./types/call_mode"; const CALL_MODE_KEY = "mode"; -const TRANSFORM_ALGO_KEY = "algo"; - -// Instantiates a client -const dlp = new DLP.DlpServiceClient({ - keyFilename: config.servicePath, -}); -const bigqueryClient = new ConnectionServiceClient({ - keyFilename: config.servicePath, -}); - -interface Item { - value: string; -} + +admin.initializeApp(); + +const dlp = new DLP.DlpServiceClient(); +const bigqueryClient = new BigQuery(); +const bigqueryConnectionClient = new ConnectionServiceClient(); + +interface CallerRow {} interface BQRequest { requestId: string; @@ -27,24 +26,6 @@ interface BQRequest { calls: []; } -const deidentifyConfig = { - parent: parent, - deidentifyConfig: { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: "x", - numberToMask: 5, - }, - }, - }, - ], - }, - }, -}; - /** * Deidentify sensitive data in a string using the Data Loss Prevention API. * @@ -52,45 +33,69 @@ const deidentifyConfig = { * * @returns {string} The deidentified text. */ -async function deidentifyWithMask(rows: Item[]) { +async function deidentifyWithMask(rows: CallerRow[]) { const deidentifiedItems = []; - const parent = `projects/${config.projectId}/locations/${location}`; + const parent = `projects/${config.projectId}/locations/${config.location}`; + const deidentifyConfig = { + parent: parent, + deidentifyConfig: { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: "x", + numberToMask: 5, + }, + }, + }, + ], + }, + }, + }; for (const row of rows) { - const request = { - ...deidentifyConfig, - item: { value: row.value }, - parent: parent, - }; - - const [response] = await dlp.deidentifyContent(request); - deidentifiedItems.push(response.item?.value); - - console.log(response); + for (const value in row) { + const request = { + ...deidentifyConfig, + item: { value: value }, + parent: parent, + }; + const [response] = await dlp.deidentifyContent(request); + deidentifiedItems.push(response.item?.value); + + functions.logger.debug(response); + } } return deidentifiedItems; } -export const deidentifyData = functions.https.onCall(async (data, ctx) => { - const bqrequest = data as BQRequest; +export const deidentifyData = functions.https.onRequest( + async (request, response) => { + const { calls } = request.body as BQRequest; - const options = checkNotNull(bqrequest.userDefinedContext); - const callMode = identifyCallMode(options); + functions.logger.debug("Incoming request from BigQuery", request.body); - switch (callMode) { - case CallMode.DEIDENTIFY: - break; - case CallMode.REIDENTIFY: - break; - } + // const options = checkNotNull(userDefinedContext); + // const callMode = identifyCallMode(options); - const items = bqrequest.calls; - const result = await deidentifyWithMask(items); + // switch (callMode) { + // case CallMode.DEIDENTIFY: + // break; + // case CallMode.REIDENTIFY: + // break; + // } - return JSON.stringify(result); -}); + try { + const result = await deidentifyWithMask(calls); + response.send(JSON.stringify({ replies: result })); + } catch (error) { + response.status(500).send(`Something wrong happend ${error}`); + } + } +); function checkNotNull(options: BQRequest["userDefinedContext"]) { if (options == null) { @@ -113,19 +118,79 @@ function identifyCallMode(options: any): string { } } -exports.onInstall = functions.tasks.taskQueue().onDispatch(async (task) => { - const data = task.data; - console.log("Task received: ", data); +exports.createBigQueryConnection = functions.tasks + .taskQueue() + .onDispatch(async (task) => { + const runtime = getExtensions().runtime(); - const projectId = data.projectId; - const location = data.location; - const parent = `projects/${projectId}/locations/${location}`; + console.log("Task received => ", task); - const connection = bigqueryClient.createConnection({ - parent: parent, - connection: { name: "deidentifyData", cloudResource: {} }, - connectionId: "connectionId", - }); + const parent = `projects/${config.projectId}/locations/${config.location}`; - console.log("Connection: ", connection); -}); + functions.logger.debug(`${parent}/connections/deidentify`); + + try { + const connection1 = await bigqueryConnectionClient.createConnection({ + parent: parent, + connectionId: `${parent}/connections/deidentify`, + connection: { + cloudResource: { + serviceAccountId: + "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", + }, + name: "deidentify", + friendlyName: "Deidentify Data", + }, + }); + + const connection2 = await bigqueryConnectionClient.createConnection({ + parent: parent, + connectionId: `projects/${config.projectId}/locations/${config.location}/connections/reidentify`, + connection: { + cloudResource: { + serviceAccountId: + "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", + }, + name: "reidentify", + friendlyName: "Reidentify Data", + }, + }); + + functions.logger.info("Connection 1 => ", connection1); + functions.logger.info("Connection 2 => ", connection2); + + if (connection1 && connection2) { + const query = `CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deindetify(data STRING) RETURNS STRING + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.ext-bq-dlp.deidentify\` + OPTIONS ( + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + )`; + + const options = { + query: query, + location: "US", + }; + + // Run the query as a job + const [job] = await bigqueryClient.createQueryJob(options); + functions.logger.debug(`Job ${job.id} started.`); + + // Wait for the query to finish + const [rows] = await job.getQueryResults(); + + functions.logger.debug("Rows: ", rows); + + await runtime.setProcessingState( + "PROCESSING_COMPLETE", + "Connections created successfully." + ); + } + } catch (error) { + functions.logger.error(error); + + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Connections were not created." + ); + } + }); From e2f15c6c9a52a68721c859e811bc9dcc6103359e Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 9 Dec 2022 23:03:46 +0300 Subject: [PATCH 04/37] fix the connectionId & add a new role --- bigquery-dlp-function/extension.yaml | 4 ++-- bigquery-dlp-function/functions/lib/index.js | 7 +++---- bigquery-dlp-function/functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 8 +++----- 4 files changed, 9 insertions(+), 12 deletions(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 66dc3a5a..cf6ceb5f 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -26,8 +26,8 @@ apis: reason: Allows the extension to create a BigQuery connection. roles: - # - role: actions.Viewer - # reason: Allows the extension to use services. + - role: bigquery.jobUser + reason: Allows the extension to create BigQuery jobs. - role: bigquery.connectionAdmin reason: Allows the extension to create a BigQuery connection. diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index c3a30b88..cdd2575b 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -93,11 +93,10 @@ exports.createBigQueryConnection = functions.tasks const runtime = (0, extensions_1.getExtensions)().runtime(); console.log("Task received => ", task); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - functions.logger.debug(`${parent}/connections/deidentify`); try { const connection1 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `${parent}/connections/deidentify`, + connectionId: `deidentify`, connection: { cloudResource: { serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", @@ -108,7 +107,7 @@ exports.createBigQueryConnection = functions.tasks }); const connection2 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `projects/${config_1.default.projectId}/locations/${config_1.default.location}/connections/reidentify`, + connectionId: `reidentify`, connection: { cloudResource: { serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", @@ -127,7 +126,7 @@ exports.createBigQueryConnection = functions.tasks )`; const options = { query: query, - location: "US", + location: config_1.default.location, }; // Run the query as a job const [job] = await bigqueryClient.createQueryJob(options); diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 9cc4f5ef..e73b877b 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,iDAAyC;AAEzC,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAY/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,oDAAoD;IACpD,8CAA8C;IAE9C,sBAAsB;IACtB,8BAA8B;IAC9B,aAAa;IACb,8BAA8B;IAC9B,aAAa;IACb,IAAI;IAEJ,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;KACpD;IAAC,OAAO,KAAK,EAAE;QACd,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;KAC/D;AACH,CAAC,CACF,CAAC;AAEF,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,mBAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,GAAG,MAAM,yBAAyB,CAAC,CAAC;IAE3D,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,MAAM,yBAAyB;YAChD,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,yBAAyB;YAChG,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACvD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAEvD,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG,qBAAqB,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;iCAC9C,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ;;8BAEtC,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;QACzD,CAAC;YAED,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,IAAI;aACf,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,iDAAyC;AAEzC,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAY/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,oDAAoD;IACpD,8CAA8C;IAE9C,sBAAsB;IACtB,8BAA8B;IAC9B,aAAa;IACb,8BAA8B;IAC9B,aAAa;IACb,IAAI;IAEJ,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;KACpD;IAAC,OAAO,KAAK,EAAE;QACd,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;KAC/D;AACH,CAAC,CACF,CAAC;AAEF,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,mBAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACvD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAEvD,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG,qBAAqB,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;iCAC9C,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ;;8BAEtC,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;QACzD,CAAC;YAED,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 723a30aa..1278b740 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -127,12 +127,10 @@ exports.createBigQueryConnection = functions.tasks const parent = `projects/${config.projectId}/locations/${config.location}`; - functions.logger.debug(`${parent}/connections/deidentify`); - try { const connection1 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `${parent}/connections/deidentify`, + connectionId: `deidentify`, connection: { cloudResource: { serviceAccountId: @@ -145,7 +143,7 @@ exports.createBigQueryConnection = functions.tasks const connection2 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `projects/${config.projectId}/locations/${config.location}/connections/reidentify`, + connectionId: `reidentify`, connection: { cloudResource: { serviceAccountId: @@ -168,7 +166,7 @@ exports.createBigQueryConnection = functions.tasks const options = { query: query, - location: "US", + location: config.location, }; // Run the query as a job From 5d8b17e0a324739f6561b5f68334d0b19e077aad Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Mon, 12 Dec 2022 17:19:41 +0300 Subject: [PATCH 05/37] fix: creating functions query --- bigquery-dlp-function/extension.yaml | 6 ++- bigquery-dlp-function/functions/lib/index.js | 37 ++++++++++------ .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 42 +++++++++++++------ 4 files changed, 61 insertions(+), 26 deletions(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index cf6ceb5f..51763674 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -24,10 +24,14 @@ apis: reason: Powers all BigQuery tasks performed by the extension. - apiName: bigqueryconnection.googleapis.com reason: Allows the extension to create a BigQuery connection. + - apiName: dlp.googleapis.com + reason: Allows the extension to use DLP services. roles: - role: bigquery.jobUser reason: Allows the extension to create BigQuery jobs. + - role: bigquery.dataOwner + reason: Allows the extension to create BigQuery routines. - role: bigquery.connectionAdmin reason: Allows the extension to create a BigQuery connection. @@ -109,4 +113,4 @@ params: lifecycleEvents: onInstall: function: createBigQueryConnection - processingMessage: "Create BigQuery connections" + processingMessage: "Creating BigQuery connections" diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index cdd2575b..89630ae5 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -64,11 +64,14 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => // break; // } try { - const result = await deidentifyWithMask(calls); - response.send(JSON.stringify({ replies: result })); + const bqResponse = { + replies: await deidentifyWithMask(calls), + }; + response.send(bqResponse); } catch (error) { - response.status(500).send(`Something wrong happend ${error}`); + functions.logger.error(error); + response.status(500).send(`errorMessage: ${error}`); } }); function checkNotNull(options) { @@ -93,37 +96,47 @@ exports.createBigQueryConnection = functions.tasks const runtime = (0, extensions_1.getExtensions)().runtime(); console.log("Task received => ", task); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + const connectionIdPrefix = `ext-bq-dlp-`; try { const connection1 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `deidentify`, + connectionId: `${connectionIdPrefix}deidentify`, connection: { cloudResource: { serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", }, - name: "deidentify", + name: `${connectionIdPrefix}deidentify`, friendlyName: "Deidentify Data", }, }); const connection2 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `reidentify`, + connectionId: `${connectionIdPrefix}reidentify`, connection: { cloudResource: { serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", }, - name: "reidentify", + name: `${connectionIdPrefix}reidentify`, friendlyName: "Reidentify Data", }, }); functions.logger.info("Connection 1 => ", connection1); functions.logger.info("Connection 2 => ", connection2); if (connection1 && connection2) { - const query = `CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deindetify(data STRING) RETURNS STRING - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.ext-bq-dlp.deidentify\` - OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' - )`; + const query = ` + BEGIN + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deindetify(data JSON) RETURNS JSON + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}deidentify\` + OPTIONS ( + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + ); + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}reidentify\` + OPTIONS ( + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + ); + END; + `; const options = { query: query, location: config_1.default.location, diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index e73b877b..cdfd6ee9 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,iDAAyC;AAEzC,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAY/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,oDAAoD;IACpD,8CAA8C;IAE9C,sBAAsB;IACtB,8BAA8B;IAC9B,aAAa;IACb,8BAA8B;IAC9B,aAAa;IACb,IAAI;IAEJ,IAAI;QACF,MAAM,MAAM,GAAG,MAAM,kBAAkB,CAAC,KAAK,CAAC,CAAC;QAC/C,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC;KACpD;IAAC,OAAO,KAAK,EAAE;QACd,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,2BAA2B,KAAK,EAAE,CAAC,CAAC;KAC/D;AACH,CAAC,CACF,CAAC;AAEF,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,mBAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACvD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAEvD,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG,qBAAqB,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;iCAC9C,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ;;8BAEtC,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;QACzD,CAAC;YAED,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,iDAAyC;AAEzC,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAgB/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,oDAAoD;IACpD,8CAA8C;IAE9C,sBAAsB;IACtB,8BAA8B;IAC9B,aAAa;IACb,8BAA8B;IAC9B,aAAa;IACb,IAAI;IAEJ,IAAI;QACF,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,mBAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACvD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAEvD,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 1278b740..eda8110e 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -26,6 +26,10 @@ interface BQRequest { calls: []; } +interface BQResponse { + replies?: (null | undefined | string)[]; +} + /** * Deidentify sensitive data in a string using the Data Loss Prevention API. * @@ -89,10 +93,14 @@ export const deidentifyData = functions.https.onRequest( // } try { - const result = await deidentifyWithMask(calls); - response.send(JSON.stringify({ replies: result })); + const bqResponse: BQResponse = { + replies: await deidentifyWithMask(calls), + }; + response.send(bqResponse); } catch (error) { - response.status(500).send(`Something wrong happend ${error}`); + functions.logger.error(error); + + response.status(500).send(`errorMessage: ${error}`); } } ); @@ -126,30 +134,31 @@ exports.createBigQueryConnection = functions.tasks console.log("Task received => ", task); const parent = `projects/${config.projectId}/locations/${config.location}`; + const connectionIdPrefix = `ext-bq-dlp-`; try { const connection1 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `deidentify`, + connectionId: `${connectionIdPrefix}deidentify`, connection: { cloudResource: { serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", }, - name: "deidentify", + name: `${connectionIdPrefix}deidentify`, friendlyName: "Deidentify Data", }, }); const connection2 = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `reidentify`, + connectionId: `${connectionIdPrefix}reidentify`, connection: { cloudResource: { serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", }, - name: "reidentify", + name: `${connectionIdPrefix}reidentify`, friendlyName: "Reidentify Data", }, }); @@ -158,11 +167,20 @@ exports.createBigQueryConnection = functions.tasks functions.logger.info("Connection 2 => ", connection2); if (connection1 && connection2) { - const query = `CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deindetify(data STRING) RETURNS STRING - REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.ext-bq-dlp.deidentify\` - OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' - )`; + const query = ` + BEGIN + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deindetify(data JSON) RETURNS JSON + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}deidentify\` + OPTIONS ( + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + ); + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data JSON) RETURNS JSON + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}reidentify\` + OPTIONS ( + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + ); + END; + `; const options = { query: query, From f385abd58d963c498e6ae8fd2a721322e5a69989 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Tue, 13 Dec 2022 12:05:21 +0300 Subject: [PATCH 06/37] chore: remove unused code --- bigquery-dlp-function/functions/lib/index.js | 31 +------------- .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 40 ++----------------- .../functions/src/types/call_mode.ts | 6 --- 4 files changed, 6 insertions(+), 73 deletions(-) delete mode 100644 bigquery-dlp-function/functions/src/types/call_mode.ts diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 89630ae5..615ce16a 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -1,6 +1,5 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.deidentifyData = void 0; const functions = require("firebase-functions"); const admin = require("firebase-admin"); const dlp_1 = require("@google-cloud/dlp"); @@ -8,8 +7,6 @@ const bigquery_connection_1 = require("@google-cloud/bigquery-connection"); const bigquery_1 = require("@google-cloud/bigquery"); const extensions_1 = require("firebase-admin/extensions"); const config_1 = require("./config"); -const call_mode_1 = require("./types/call_mode"); -const CALL_MODE_KEY = "mode"; admin.initializeApp(); const dlp = new dlp_1.default.DlpServiceClient(); const bigqueryClient = new bigquery_1.BigQuery(); @@ -55,14 +52,6 @@ async function deidentifyWithMask(rows) { exports.deidentifyData = functions.https.onRequest(async (request, response) => { const { calls } = request.body; functions.logger.debug("Incoming request from BigQuery", request.body); - // const options = checkNotNull(userDefinedContext); - // const callMode = identifyCallMode(options); - // switch (callMode) { - // case CallMode.DEIDENTIFY: - // break; - // case CallMode.REIDENTIFY: - // break; - // } try { const bqResponse = { replies: await deidentifyWithMask(calls), @@ -74,22 +63,6 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => response.status(500).send(`errorMessage: ${error}`); } }); -function checkNotNull(options) { - if (options == null) { - throw new Error("userDefinedContext is required. Found null."); - } - return { replies: options }; -} -function identifyCallMode(options) { - var callMode = CALL_MODE_KEY in options ? options[CALL_MODE_KEY] : null; - if (CALL_MODE_KEY in options) { - const callModeEnum = callMode; - return callModeEnum; - } - else { - return call_mode_1.default.DEIDENTIFY; - } -} exports.createBigQueryConnection = functions.tasks .taskQueue() .onDispatch(async (task) => { @@ -120,8 +93,8 @@ exports.createBigQueryConnection = functions.tasks friendlyName: "Reidentify Data", }, }); - functions.logger.info("Connection 1 => ", connection1); - functions.logger.info("Connection 2 => ", connection2); + functions.logger.info("Connection 1 created => ", connection1); + functions.logger.info("Connection 2 created => ", connection2); if (connection1 && connection2) { const query = ` BEGIN diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index cdfd6ee9..28f6bfb5 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,iDAAyC;AAEzC,MAAM,aAAa,GAAG,MAAM,CAAC;AAE7B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAgB/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAEY,QAAA,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACrD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,oDAAoD;IACpD,8CAA8C;IAE9C,sBAAsB;IACtB,8BAA8B;IAC9B,aAAa;IACb,8BAA8B;IAC9B,aAAa;IACb,IAAI;IAEJ,IAAI;QACF,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,SAAS,YAAY,CAAC,OAAwC;IAC5D,IAAI,OAAO,IAAI,IAAI,EAAE;QACnB,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;KAChE;IAED,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC9B,CAAC;AAED,SAAS,gBAAgB,CAAC,OAAY;IACpC,IAAI,QAAQ,GACV,aAAa,IAAI,OAAO,CAAC,CAAC,CAAE,OAAO,CAAC,aAAa,CAAY,CAAC,CAAC,CAAC,IAAI,CAAC;IAEvE,IAAI,aAAa,IAAI,OAAO,EAAE;QAC5B,MAAM,YAAY,GAAG,QAAoB,CAAC;QAE1C,OAAO,YAAY,CAAC;KACrB;SAAM;QACL,OAAO,mBAAQ,CAAC,UAAU,CAAC;KAC5B;AACH,CAAC;AAED,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QACvD,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,kBAAkB,EAAE,WAAW,CAAC,CAAC;QAEvD,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAE9B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAgB/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,IAAI;QACF,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC/D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAE/D,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index eda8110e..16357056 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -6,9 +6,6 @@ import { BigQuery } from "@google-cloud/bigquery"; import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; -import CallMode from "./types/call_mode"; - -const CALL_MODE_KEY = "mode"; admin.initializeApp(); @@ -76,22 +73,12 @@ async function deidentifyWithMask(rows: CallerRow[]) { return deidentifiedItems; } -export const deidentifyData = functions.https.onRequest( +exports.deidentifyData = functions.https.onRequest( async (request, response) => { const { calls } = request.body as BQRequest; functions.logger.debug("Incoming request from BigQuery", request.body); - // const options = checkNotNull(userDefinedContext); - // const callMode = identifyCallMode(options); - - // switch (callMode) { - // case CallMode.DEIDENTIFY: - // break; - // case CallMode.REIDENTIFY: - // break; - // } - try { const bqResponse: BQResponse = { replies: await deidentifyWithMask(calls), @@ -105,27 +92,6 @@ export const deidentifyData = functions.https.onRequest( } ); -function checkNotNull(options: BQRequest["userDefinedContext"]) { - if (options == null) { - throw new Error("userDefinedContext is required. Found null."); - } - - return { replies: options }; -} - -function identifyCallMode(options: any): string { - var callMode = - CALL_MODE_KEY in options ? (options[CALL_MODE_KEY] as string) : null; - - if (CALL_MODE_KEY in options) { - const callModeEnum = callMode as CallMode; - - return callModeEnum; - } else { - return CallMode.DEIDENTIFY; - } -} - exports.createBigQueryConnection = functions.tasks .taskQueue() .onDispatch(async (task) => { @@ -163,8 +129,8 @@ exports.createBigQueryConnection = functions.tasks }, }); - functions.logger.info("Connection 1 => ", connection1); - functions.logger.info("Connection 2 => ", connection2); + functions.logger.info("Connection 1 created => ", connection1); + functions.logger.info("Connection 2 created => ", connection2); if (connection1 && connection2) { const query = ` diff --git a/bigquery-dlp-function/functions/src/types/call_mode.ts b/bigquery-dlp-function/functions/src/types/call_mode.ts deleted file mode 100644 index 37041655..00000000 --- a/bigquery-dlp-function/functions/src/types/call_mode.ts +++ /dev/null @@ -1,6 +0,0 @@ -enum CallMode { - DEIDENTIFY = "DEIDENTIFY", - REIDENTIFY = "REIDENTIFY", -} - -export default CallMode; From 601ca694ae2eb01a0f219033fb8e51aa0b8bdd01 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 14 Dec 2022 18:21:34 +0300 Subject: [PATCH 07/37] fix: dlp role --- bigquery-dlp-function/extension.yaml | 2 + bigquery-dlp-function/functions/lib/index.js | 21 +++++------ .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 37 +++++++++---------- 4 files changed, 31 insertions(+), 31 deletions(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 51763674..94cabd6e 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -34,6 +34,8 @@ roles: reason: Allows the extension to create BigQuery routines. - role: bigquery.connectionAdmin reason: Allows the extension to create a BigQuery connection. + - role: dlp.user + reason: Allows the extension to use DLP services. resources: - name: createBigQueryConnection diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 615ce16a..10da7a03 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -31,7 +31,6 @@ async function deidentifyWithMask(rows) { primitiveTransformation: { characterMaskConfig: { maskingCharacter: "x", - numberToMask: 5, }, }, }, @@ -40,18 +39,18 @@ async function deidentifyWithMask(rows) { }, }; for (const row of rows) { - for (const value in row) { - const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: value }, parent: parent }); - const [response] = await dlp.deidentifyContent(request); - deidentifiedItems.push((_a = response.item) === null || _a === void 0 ? void 0 : _a.value); - functions.logger.debug(response); - } + const data = row[0]; + functions.logger.debug(data); + const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: data }, parent: parent }); + const [response] = await dlp.deidentifyContent(request); + deidentifiedItems.push((_a = response.item) === null || _a === void 0 ? void 0 : _a.value); + functions.logger.debug(response.item); } return deidentifiedItems; } exports.deidentifyData = functions.https.onRequest(async (request, response) => { const { calls } = request.body; - functions.logger.debug("Incoming request from BigQuery", request.body); + functions.logger.debug("Incoming request from BigQuery", calls); try { const bqResponse = { replies: await deidentifyWithMask(calls), @@ -76,7 +75,7 @@ exports.createBigQueryConnection = functions.tasks connectionId: `${connectionIdPrefix}deidentify`, connection: { cloudResource: { - serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", + serviceAccountId: `ext-bigquery-dlp-function@${config_1.default.projectId}.iam.gserviceaccount.com`, }, name: `${connectionIdPrefix}deidentify`, friendlyName: "Deidentify Data", @@ -98,12 +97,12 @@ exports.createBigQueryConnection = functions.tasks if (connection1 && connection2) { const query = ` BEGIN - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deindetify(data JSON) RETURNS JSON + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data STRING) RETURNS STRING REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}deidentify\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' ); - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data STRING) RETURNS STRING REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}reidentify\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 28f6bfb5..da9923d4 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAE9B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAgB/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAiB;;IACjD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;gCACrB,YAAY,EAAE,CAAC;6BAChB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,KAAK,MAAM,KAAK,IAAI,GAAG,EAAE;YACvB,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,EACtB,MAAM,EAAE,MAAM,GACf,CAAC;YACF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;YACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;YAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;SAClC;KACF;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAiB,CAAC;IAE5C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC;IAEvE,IAAI;QACF,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC/D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAE/D,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAE9B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAc/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAQ;;IACxC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;6BACtB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAO,IAAI,GAAG,GAAG,CAAC,CAAC,CAAW,CAAC;QAC/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EACrB,MAAM,EAAE,MAAM,GACf,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;QAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;KACvC;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAChB,6BAA6B,gBAAM,CAAC,SAAS,0BAA0B;iBACxE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC/D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAE/D,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 16357056..68d3b8cb 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -13,8 +13,6 @@ const dlp = new DLP.DlpServiceClient(); const bigqueryClient = new BigQuery(); const bigqueryConnectionClient = new ConnectionServiceClient(); -interface CallerRow {} - interface BQRequest { requestId: string; callecallerrIp: string; @@ -34,7 +32,7 @@ interface BQResponse { * * @returns {string} The deidentified text. */ -async function deidentifyWithMask(rows: CallerRow[]) { +async function deidentifyWithMask(rows:any) { const deidentifiedItems = []; const parent = `projects/${config.projectId}/locations/${config.location}`; @@ -47,7 +45,6 @@ async function deidentifyWithMask(rows: CallerRow[]) { primitiveTransformation: { characterMaskConfig: { maskingCharacter: "x", - numberToMask: 5, }, }, }, @@ -57,17 +54,19 @@ async function deidentifyWithMask(rows: CallerRow[]) { }; for (const row of rows) { - for (const value in row) { - const request = { - ...deidentifyConfig, - item: { value: value }, - parent: parent, - }; - const [response] = await dlp.deidentifyContent(request); - deidentifiedItems.push(response.item?.value); + const data = row[0] as string; + functions.logger.debug(data); - functions.logger.debug(response); - } + const request = { + ...deidentifyConfig, + item: { value: data }, + parent: parent, + }; + + const [response] = await dlp.deidentifyContent(request); + deidentifiedItems.push(response.item?.value); + + functions.logger.debug(response.item); } return deidentifiedItems; @@ -75,9 +74,9 @@ async function deidentifyWithMask(rows: CallerRow[]) { exports.deidentifyData = functions.https.onRequest( async (request, response) => { - const { calls } = request.body as BQRequest; + const { calls } = request.body; - functions.logger.debug("Incoming request from BigQuery", request.body); + functions.logger.debug("Incoming request from BigQuery", calls); try { const bqResponse: BQResponse = { @@ -109,7 +108,7 @@ exports.createBigQueryConnection = functions.tasks connection: { cloudResource: { serviceAccountId: - "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", + `ext-bigquery-dlp-function@${config.projectId}.iam.gserviceaccount.com`, }, name: `${connectionIdPrefix}deidentify`, friendlyName: "Deidentify Data", @@ -135,12 +134,12 @@ exports.createBigQueryConnection = functions.tasks if (connection1 && connection2) { const query = ` BEGIN - CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deindetify(data JSON) RETURNS JSON + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data STRING) RETURNS STRING REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}deidentify\` OPTIONS ( endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' ); - CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data JSON) RETURNS JSON + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data STRING) RETURNS STRING REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}reidentify\` OPTIONS ( endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' From 8eca3cc0aa8173d0afd84b4913bcb4184619dee6 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Thu, 15 Dec 2022 15:37:54 +0300 Subject: [PATCH 08/37] feat: JSON approach --- bigquery-dlp-function/functions/lib/index.js | 23 ++++++----- .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 38 ++++++++++--------- 3 files changed, 34 insertions(+), 29 deletions(-) diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 10da7a03..28056841 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -41,10 +41,16 @@ async function deidentifyWithMask(rows) { for (const row of rows) { const data = row[0]; functions.logger.debug(data); - const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: data }, parent: parent }); - const [response] = await dlp.deidentifyContent(request); - deidentifiedItems.push((_a = response.item) === null || _a === void 0 ? void 0 : _a.value); - functions.logger.debug(response.item); + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: element }, parent: parent }); + const [response] = await dlp.deidentifyContent(request); + data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; + } + } + functions.logger.debug(data); + deidentifiedItems.push(data); } return deidentifiedItems; } @@ -52,10 +58,7 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => const { calls } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); try { - const bqResponse = { - replies: await deidentifyWithMask(calls), - }; - response.send(bqResponse); + response.send({ replies: await deidentifyWithMask(calls) }); } catch (error) { functions.logger.error(error); @@ -97,12 +100,12 @@ exports.createBigQueryConnection = functions.tasks if (connection1 && connection2) { const query = ` BEGIN - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data STRING) RETURNS STRING + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}deidentify\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' ); - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data STRING) RETURNS STRING + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}reidentify\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index da9923d4..2ceecb4f 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAE9B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAc/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAQ;;IACxC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;6BACtB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAO,IAAI,GAAG,GAAG,CAAC,CAAC,CAAW,CAAC;QAC/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,EACrB,MAAM,EAAE,MAAM,GACf,CAAC;QAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;QACxD,iBAAiB,CAAC,IAAI,CAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;QAE7C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;KACvC;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,MAAM,UAAU,GAAe;YAC7B,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC;SACzC,CAAC;QACF,QAAQ,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAChB,6BAA6B,gBAAM,CAAC,SAAS,0BAA0B;iBACxE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC/D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAE/D,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAE9B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAc/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAS;;IACzC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;6BACtB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACxD,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;KAC7D;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,6BAA6B,gBAAM,CAAC,SAAS,0BAA0B;iBAC1F;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC/D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAE/D,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 68d3b8cb..289f5845 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -32,7 +32,7 @@ interface BQResponse { * * @returns {string} The deidentified text. */ -async function deidentifyWithMask(rows:any) { +async function deidentifyWithMask(rows: any) { const deidentifiedItems = []; const parent = `projects/${config.projectId}/locations/${config.location}`; @@ -54,19 +54,25 @@ async function deidentifyWithMask(rows:any) { }; for (const row of rows) { - const data = row[0] as string; + const data = row[0] as Record; functions.logger.debug(data); - const request = { - ...deidentifyConfig, - item: { value: data }, - parent: parent, - }; + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = { + ...deidentifyConfig, + item: { value: element }, + parent: parent, + }; - const [response] = await dlp.deidentifyContent(request); - deidentifiedItems.push(response.item?.value); + const [response] = await dlp.deidentifyContent(request); + data[key] = response.item?.value; + } + } - functions.logger.debug(response.item); + functions.logger.debug(data); + deidentifiedItems.push(data); } return deidentifiedItems; @@ -79,10 +85,7 @@ exports.deidentifyData = functions.https.onRequest( functions.logger.debug("Incoming request from BigQuery", calls); try { - const bqResponse: BQResponse = { - replies: await deidentifyWithMask(calls), - }; - response.send(bqResponse); + response.send({ replies: await deidentifyWithMask(calls) }); } catch (error) { functions.logger.error(error); @@ -107,8 +110,7 @@ exports.createBigQueryConnection = functions.tasks connectionId: `${connectionIdPrefix}deidentify`, connection: { cloudResource: { - serviceAccountId: - `ext-bigquery-dlp-function@${config.projectId}.iam.gserviceaccount.com`, + serviceAccountId: `ext-bigquery-dlp-function@${config.projectId}.iam.gserviceaccount.com`, }, name: `${connectionIdPrefix}deidentify`, friendlyName: "Deidentify Data", @@ -134,12 +136,12 @@ exports.createBigQueryConnection = functions.tasks if (connection1 && connection2) { const query = ` BEGIN - CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data STRING) RETURNS STRING + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}deidentify\` OPTIONS ( endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' ); - CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data STRING) RETURNS STRING + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}reidentify\` OPTIONS ( endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' From 30bd5ba0cdc86e4901b888f4949decdc30398644 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Mon, 19 Dec 2022 16:23:05 +0300 Subject: [PATCH 09/37] feat: record type transformation --- bigquery-dlp-function/extension.yaml | 12 ++ bigquery-dlp-function/functions/src/config.ts | 1 + .../functions/src/deidentify.ts | 157 ++++++++++++++++++ bigquery-dlp-function/functions/src/index.ts | 116 +++---------- .../functions/src/reidentify.ts | 0 5 files changed, 196 insertions(+), 90 deletions(-) create mode 100644 bigquery-dlp-function/functions/src/deidentify.ts create mode 100644 bigquery-dlp-function/functions/src/reidentify.ts diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 94cabd6e..8f8c5e81 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -54,6 +54,17 @@ resources: httpsTrigger: {} params: + - param: TRANSFORMATION_METHOD + label: DLP Transformation Method + description: >- + The method used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. + type: select + options: + - label: Info Type Transformations + value: INFO_TYPE + - label: Record Type Transformations + value: RECORD + required: true - param: DATASET_ID label: BigQuery Dataset ID description: >- @@ -61,6 +72,7 @@ params: type: string default: deidentified_data required: true + immutable: true - param: LOCATION label: Cloud Functions location description: >- diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 4f63d0c0..7df3f2a0 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -18,4 +18,5 @@ export default { projectId: process.env.PROJECT_ID, datasetId: process.env.DATASET_ID, location: process.env.LOCATION, + method: process.env.TRANSFORMATION_METHOD, }; diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts new file mode 100644 index 00000000..a0dbb9fe --- /dev/null +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -0,0 +1,157 @@ +import * as functions from "firebase-functions"; +import { DlpServiceClient, protos } from "@google-cloud/dlp"; + +import config from "./config"; + +type DeidentifyRequest = protos.google.privacy.dlp.v2.IDeidentifyContentRequest; +// The maximum number of days to shift a date backward +const lowerBoundDays = 1; +// The maximum number of days to shift a date forward +const upperBoundDays = 1; + +function rowsToTable(rows: []) { + // Helper function for converting rows to Protobuf types + const rowToProto = (row: any) => { + const values = row.split(","); + const convertedValues = values.map((value: any) => { + if (Date.parse(value)) { + const date = new Date(value); + return { + dateValue: { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + }, + }; + } else { + // Convert all non-date values to strings + return { stringValue: value.toString() }; + } + }); + return { values: convertedValues }; + }; + + // Construct the table object + const table = { + headers: [], + rows: rows.map((row) => rowToProto(row[0])), + }; + + return table; +} + +/** + * Deidentify sensitive data in a string with [the Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) + * using `infoTypeTransformations` method. + * + * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#infotype_transformations + * + * @param {string} text The text to deidentify. + * + * @returns {string} The deidentified text. + */ +export async function deidentifyWithInfoTypeTransformations( + rows: any, + client: DlpServiceClient, + mask?: string, + numberToMask?: number +) { + const deidentifiedItems = []; + + const parent = `projects/${config.projectId}/locations/${config.location}`; + const deidentifyConfig: DeidentifyRequest = { + parent: parent, + deidentifyConfig: { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }, + }; + + for (const row of rows) { + const data = row[0] as Record; + functions.logger.debug(data); + + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = { + ...deidentifyConfig, + item: { value: element }, + parent: parent, + }; + + const [response] = await client.deidentifyContent(request); + data[key] = response.item?.value; + } + } + + functions.logger.debug(data); + deidentifiedItems.push(data); + } + + return deidentifiedItems; +} + +/** + * Deidentify sensitive data in a string with the [Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) + * using `recordTransformations` method. + * + * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#record_transformations + * + * @param {string} text The text to deidentify. + * + * @returns {string} The deidentified text. + */ +export async function deidentifyWithRecordTransformations( + rows: any, + client: DlpServiceClient, + mask?: string, + numberToMask?: number +) { + const parent = `projects/${config.projectId}/locations/${config.location}`; + + // Construct DateShiftConfig + const dateShiftConfig = { + lowerBoundDays: lowerBoundDays, + upperBoundDays: upperBoundDays, + }; + + // Construct de-identification request + const request: DeidentifyRequest = { + parent: parent, + deidentifyConfig: { + recordTransformations: { + fieldTransformations: [ + { + primitiveTransformation: { + dateShiftConfig: dateShiftConfig, + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }, + item: { + table: rowsToTable(rows), + }, + }; + + // Run deidentification request + const [response] = await client.deidentifyContent(request); + const tableRows = response.item?.table?.rows; + + return tableRows; +} diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 289f5845..98c737a7 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -6,6 +6,10 @@ import { BigQuery } from "@google-cloud/bigquery"; import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; +import { + deidentifyWithInfoTypeTransformations, + deidentifyWithRecordTransformations, +} from "./deidentify"; admin.initializeApp(); @@ -13,79 +17,24 @@ const dlp = new DLP.DlpServiceClient(); const bigqueryClient = new BigQuery(); const bigqueryConnectionClient = new ConnectionServiceClient(); -interface BQRequest { - requestId: string; - callecallerrIp: string; - sessionUser: string; - userDefinedContext: {}; - calls: []; -} - -interface BQResponse { - replies?: (null | undefined | string)[]; -} - -/** - * Deidentify sensitive data in a string using the Data Loss Prevention API. - * - * @param {string} text The text to deidentify. - * - * @returns {string} The deidentified text. - */ -async function deidentifyWithMask(rows: any) { - const deidentifiedItems = []; - - const parent = `projects/${config.projectId}/locations/${config.location}`; - const deidentifyConfig = { - parent: parent, - deidentifyConfig: { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: "x", - }, - }, - }, - ], - }, - }, - }; - - for (const row of rows) { - const data = row[0] as Record; - functions.logger.debug(data); - - for (const key in data) { - if (data.hasOwnProperty(key)) { - const element = data[key]; - const request = { - ...deidentifyConfig, - item: { value: element }, - parent: parent, - }; - - const [response] = await dlp.deidentifyContent(request); - data[key] = response.item?.value; - } - } - - functions.logger.debug(data); - deidentifiedItems.push(data); - } - - return deidentifiedItems; -} - exports.deidentifyData = functions.https.onRequest( async (request, response) => { - const { calls } = request.body; + const { calls, userDefinedContext } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); try { - response.send({ replies: await deidentifyWithMask(calls) }); + if (userDefinedContext.method === "INFO_TYPE") { + response.send({ + replies: await deidentifyWithInfoTypeTransformations(calls, dlp), + }); + } else if (userDefinedContext.method === "RECORD") { + response.send({ + replies: await deidentifyWithRecordTransformations(calls, dlp), + }); + } else { + response.status(400).send("Invalid method"); + } } catch (error) { functions.logger.error(error); @@ -102,47 +51,34 @@ exports.createBigQueryConnection = functions.tasks console.log("Task received => ", task); const parent = `projects/${config.projectId}/locations/${config.location}`; - const connectionIdPrefix = `ext-bq-dlp-`; + const connectionId = `ext-bigquery-dlp-function`; try { - const connection1 = await bigqueryConnectionClient.createConnection({ + const connection = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `${connectionIdPrefix}deidentify`, + connectionId: connectionId, connection: { cloudResource: { serviceAccountId: `ext-bigquery-dlp-function@${config.projectId}.iam.gserviceaccount.com`, }, - name: `${connectionIdPrefix}deidentify`, + name: connectionId, friendlyName: "Deidentify Data", }, }); - const connection2 = await bigqueryConnectionClient.createConnection({ - parent: parent, - connectionId: `${connectionIdPrefix}reidentify`, - connection: { - cloudResource: { - serviceAccountId: - "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", - }, - name: `${connectionIdPrefix}reidentify`, - friendlyName: "Reidentify Data", - }, - }); + functions.logger.info("Connection 1 created => ", connection); - functions.logger.info("Connection 1 created => ", connection1); - functions.logger.info("Connection 2 created => ", connection2); - - if (connection1 && connection2) { + if (connection) { const query = ` BEGIN CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}deidentify\` + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + user_defined_context = [("method", "${config.method}")] ); CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionIdPrefix}reidentify\` + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' ); diff --git a/bigquery-dlp-function/functions/src/reidentify.ts b/bigquery-dlp-function/functions/src/reidentify.ts new file mode 100644 index 00000000..e69de29b From 34a0820c1e76faf1e335d9033e7ff847184a40e7 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 21 Dec 2022 14:56:30 +0300 Subject: [PATCH 10/37] chore: todo fo supporting record transformation --- bigquery-dlp-function/extension.yaml | 5 +- bigquery-dlp-function/functions/lib/config.js | 1 + .../functions/lib/config.js.map | 2 +- .../functions/lib/deidentify.js | 133 ++++++++++++++++++ .../functions/lib/deidentify.js.map | 1 + bigquery-dlp-function/functions/lib/index.js | 91 ++++-------- .../functions/lib/index.js.map | 2 +- .../functions/lib/reidentify.js | 2 + .../functions/lib/reidentify.js.map | 1 + .../functions/src/deidentify.ts | 2 + 10 files changed, 170 insertions(+), 70 deletions(-) create mode 100644 bigquery-dlp-function/functions/lib/deidentify.js create mode 100644 bigquery-dlp-function/functions/lib/deidentify.js.map create mode 100644 bigquery-dlp-function/functions/lib/reidentify.js create mode 100644 bigquery-dlp-function/functions/lib/reidentify.js.map diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 8f8c5e81..dd1f4b6e 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -62,8 +62,9 @@ params: options: - label: Info Type Transformations value: INFO_TYPE - - label: Record Type Transformations - value: RECORD + # TODO support record type transformations + # - label: Record Type Transformations + # value: RECORD required: true - param: DATASET_ID label: BigQuery Dataset ID diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js index 3b962d0a..cab000fe 100644 --- a/bigquery-dlp-function/functions/lib/config.js +++ b/bigquery-dlp-function/functions/lib/config.js @@ -19,5 +19,6 @@ exports.default = { projectId: process.env.PROJECT_ID, datasetId: process.env.DATASET_ID, location: process.env.LOCATION, + method: process.env.TRANSFORMATION_METHOD, }; //# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map index 69b88d26..b66ca980 100644 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -1 +1 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;CAC/B,CAAC"} \ No newline at end of file +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;CAC1C,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js b/bigquery-dlp-function/functions/lib/deidentify.js new file mode 100644 index 00000000..e9bef4ae --- /dev/null +++ b/bigquery-dlp-function/functions/lib/deidentify.js @@ -0,0 +1,133 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.deidentifyWithRecordTransformations = exports.deidentifyWithInfoTypeTransformations = void 0; +const functions = require("firebase-functions"); +const config_1 = require("./config"); +// The maximum number of days to shift a date backward +const lowerBoundDays = 1; +// The maximum number of days to shift a date forward +const upperBoundDays = 1; +function rowsToTable(rows) { + // Helper function for converting rows to Protobuf types + const rowToProto = (row) => { + const values = row.split(","); + const convertedValues = values.map((value) => { + if (Date.parse(value)) { + const date = new Date(value); + return { + dateValue: { + year: date.getFullYear(), + month: date.getMonth() + 1, + day: date.getDate(), + }, + }; + } + else { + // Convert all non-date values to strings + return { stringValue: value.toString() }; + } + }); + return { values: convertedValues }; + }; + // Construct the table object + const table = { + headers: [], + rows: rows.map((row) => rowToProto(row[0])), + }; + return table; +} +/** + * Deidentify sensitive data in a string with [the Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) + * using `infoTypeTransformations` method. + * + * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#infotype_transformations + * + * @param {string} text The text to deidentify. + * + * @returns {string} The deidentified text. + */ +async function deidentifyWithInfoTypeTransformations(rows, client, mask, numberToMask) { + var _a; + const deidentifiedItems = []; + const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + const deidentifyConfig = { + parent: parent, + deidentifyConfig: { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", + numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, + }, + }, + }, + ], + }, + }, + }; + for (const row of rows) { + const data = row[0]; + functions.logger.debug(data); + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: element }, parent: parent }); + const [response] = await client.deidentifyContent(request); + data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; + } + } + functions.logger.debug(data); + deidentifiedItems.push(data); + } + return deidentifiedItems; +} +exports.deidentifyWithInfoTypeTransformations = deidentifyWithInfoTypeTransformations; +/** + * Deidentify sensitive data in a string with the [Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) + * using `recordTransformations` method. + * + * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#record_transformations + * + * @param {string} text The text to deidentify. + * + * @returns {string} The deidentified text. + */ +async function deidentifyWithRecordTransformations(rows, client, mask, numberToMask) { + var _a, _b; + const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + // Construct DateShiftConfig + const dateShiftConfig = { + lowerBoundDays: lowerBoundDays, + upperBoundDays: upperBoundDays, + }; + // Construct de-identification request + const request = { + parent: parent, + deidentifyConfig: { + recordTransformations: { + fieldTransformations: [ + { + primitiveTransformation: { + dateShiftConfig: dateShiftConfig, + characterMaskConfig: { + maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", + numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, + }, + }, + }, + ], + }, + }, + item: { + table: rowsToTable(rows), + }, + }; + // Run deidentification request + const [response] = await client.deidentifyContent(request); + const tableRows = (_b = (_a = response.item) === null || _a === void 0 ? void 0 : _a.table) === null || _b === void 0 ? void 0 : _b.rows; + return tableRows; +} +exports.deidentifyWithRecordTransformations = deidentifyWithRecordTransformations; +//# sourceMappingURL=deidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js.map b/bigquery-dlp-function/functions/lib/deidentify.js.map new file mode 100644 index 00000000..bf6c310b --- /dev/null +++ b/bigquery-dlp-function/functions/lib/deidentify.js.map @@ -0,0 +1 @@ +{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,qCAA8B;AAI9B,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,qDAAqD;AACrD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,SAAS,WAAW,CAAC,IAAQ;IAC3B,wDAAwD;IACxD,MAAM,UAAU,GAAG,CAAC,GAAQ,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YAChD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,OAAO;oBACL,SAAS,EAAE;wBACT,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE;wBACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBACpB;iBACF,CAAC;aACH;iBAAM;gBACL,yCAAyC;gBACzC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;aAC1C;QACH,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG;QACZ,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAS,EACT,MAAwB,EACxB,IAAa,EACb,YAAqB;;IAErB,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAsB;QAC1C,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAlDD,sFAkDC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mCAAmC,CACvD,IAAS,EACT,MAAwB,EACxB,IAAa,EACb,YAAqB;;IAErB,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,4BAA4B;IAC5B,MAAM,eAAe,GAAG;QACtB,cAAc,EAAE,cAAc;QAC9B,cAAc,EAAE,cAAc;KAC/B,CAAC;IAEF,sCAAsC;IACtC,MAAM,OAAO,GAAsB;QACjC,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,qBAAqB,EAAE;gBACrB,oBAAoB,EAAE;oBACpB;wBACE,uBAAuB,EAAE;4BACvB,eAAe,EAAE,eAAe;4BAChC,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF;QACD,IAAI,EAAE;YACJ,KAAK,EAAE,WAAW,CAAC,IAAI,CAAC;SACzB;KACF,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,MAAA,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,0CAAE,IAAI,CAAC;IAE7C,OAAO,SAAS,CAAC;AACnB,CAAC;AA1CD,kFA0CC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 28056841..d2b4c425 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -7,58 +7,28 @@ const bigquery_connection_1 = require("@google-cloud/bigquery-connection"); const bigquery_1 = require("@google-cloud/bigquery"); const extensions_1 = require("firebase-admin/extensions"); const config_1 = require("./config"); +const deidentify_1 = require("./deidentify"); admin.initializeApp(); const dlp = new dlp_1.default.DlpServiceClient(); const bigqueryClient = new bigquery_1.BigQuery(); const bigqueryConnectionClient = new bigquery_connection_1.ConnectionServiceClient(); -/** - * Deidentify sensitive data in a string using the Data Loss Prevention API. - * - * @param {string} text The text to deidentify. - * - * @returns {string} The deidentified text. - */ -async function deidentifyWithMask(rows) { - var _a; - const deidentifiedItems = []; - const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const deidentifyConfig = { - parent: parent, - deidentifyConfig: { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: "x", - }, - }, - }, - ], - }, - }, - }; - for (const row of rows) { - const data = row[0]; - functions.logger.debug(data); - for (const key in data) { - if (data.hasOwnProperty(key)) { - const element = data[key]; - const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: element }, parent: parent }); - const [response] = await dlp.deidentifyContent(request); - data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; - } - } - functions.logger.debug(data); - deidentifiedItems.push(data); - } - return deidentifiedItems; -} exports.deidentifyData = functions.https.onRequest(async (request, response) => { - const { calls } = request.body; + const { calls, userDefinedContext } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); try { - response.send({ replies: await deidentifyWithMask(calls) }); + if (userDefinedContext.method === "INFO_TYPE") { + response.send({ + replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp), + }); + } + else if (userDefinedContext.method === "RECORD") { + response.send({ + replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp), + }); + } + else { + response.status(400).send("Invalid method"); + } } catch (error) { functions.logger.error(error); @@ -71,42 +41,31 @@ exports.createBigQueryConnection = functions.tasks const runtime = (0, extensions_1.getExtensions)().runtime(); console.log("Task received => ", task); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const connectionIdPrefix = `ext-bq-dlp-`; + const connectionId = `ext-bigquery-dlp-function`; try { - const connection1 = await bigqueryConnectionClient.createConnection({ + const connection = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: `${connectionIdPrefix}deidentify`, + connectionId: connectionId, connection: { cloudResource: { serviceAccountId: `ext-bigquery-dlp-function@${config_1.default.projectId}.iam.gserviceaccount.com`, }, - name: `${connectionIdPrefix}deidentify`, + name: connectionId, friendlyName: "Deidentify Data", }, }); - const connection2 = await bigqueryConnectionClient.createConnection({ - parent: parent, - connectionId: `${connectionIdPrefix}reidentify`, - connection: { - cloudResource: { - serviceAccountId: "ext-bigquery-dlp-function@extensions-testing.iam.gserviceaccount.com", - }, - name: `${connectionIdPrefix}reidentify`, - friendlyName: "Reidentify Data", - }, - }); - functions.logger.info("Connection 1 created => ", connection1); - functions.logger.info("Connection 2 created => ", connection2); - if (connection1 && connection2) { + functions.logger.info("Connection 1 created => ", connection); + if (connection) { const query = ` BEGIN CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}deidentify\` + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + user_defined_context = [("method", "${config_1.default.method}")] ); CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionIdPrefix}reidentify\` + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' ); diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 2ceecb4f..400c66a8 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAE9B,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAc/D;;;;;;GAMG;AACH,KAAK,UAAU,kBAAkB,CAAC,IAAS;;IACzC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAG;QACvB,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,GAAG;6BACtB;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,GAAG,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBACxD,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAED,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,QAAQ,CAAC,IAAI,CAAC,EAAE,OAAO,EAAE,MAAM,kBAAkB,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;KAC7D;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,kBAAkB,GAAG,aAAa,CAAC;IAEzC,IAAI;QACF,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,6BAA6B,gBAAM,CAAC,SAAS,0BAA0B;iBAC1F;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,MAAM,WAAW,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAClE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,GAAG,kBAAkB,YAAY;YAC/C,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EACd,sEAAsE;iBACzE;gBACD,IAAI,EAAE,GAAG,kBAAkB,YAAY;gBACvC,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAC/D,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;QAE/D,IAAI,WAAW,IAAI,WAAW,EAAE;YAC9B,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;8BAEvC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,kBAAkB;;kCAE5D,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AAEtB,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAAC,KAAK,EAAE,GAAG,CAAC;aACjE,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAAC,KAAK,EAAE,GAAG,CAAC;aAC/D,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,2BAA2B,CAAC;IAEjD,IAAI;QACF,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YACjE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,6BAA6B,gBAAM,CAAC,SAAS,0BAA0B;iBAC1F;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAC;QAE9D,IAAI,UAAU,EAAE;YACd,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js b/bigquery-dlp-function/functions/lib/reidentify.js new file mode 100644 index 00000000..0781ad93 --- /dev/null +++ b/bigquery-dlp-function/functions/lib/reidentify.js @@ -0,0 +1,2 @@ +"use strict"; +//# sourceMappingURL=reidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js.map b/bigquery-dlp-function/functions/lib/reidentify.js.map new file mode 100644 index 00000000..bcf5dcce --- /dev/null +++ b/bigquery-dlp-function/functions/lib/reidentify.js.map @@ -0,0 +1 @@ +{"version":3,"file":"reidentify.js","sourceRoot":"","sources":["../src/reidentify.ts"],"names":[],"mappings":""} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index a0dbb9fe..0f4fa83b 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -4,8 +4,10 @@ import { DlpServiceClient, protos } from "@google-cloud/dlp"; import config from "./config"; type DeidentifyRequest = protos.google.privacy.dlp.v2.IDeidentifyContentRequest; + // The maximum number of days to shift a date backward const lowerBoundDays = 1; + // The maximum number of days to shift a date forward const upperBoundDays = 1; From 0b3dffcee19e3b7242eef831e6a51d263f39e40c Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 21 Dec 2022 17:52:53 +0300 Subject: [PATCH 11/37] feat: handle connection ALREADY_EXISTS --- bigquery-dlp-function/functions/lib/config.js | 1 + .../functions/lib/config.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 48 +++++++++------ .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/config.ts | 1 + bigquery-dlp-function/functions/src/index.ts | 59 +++++++++++-------- 6 files changed, 69 insertions(+), 44 deletions(-) diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js index cab000fe..557073f5 100644 --- a/bigquery-dlp-function/functions/lib/config.js +++ b/bigquery-dlp-function/functions/lib/config.js @@ -17,6 +17,7 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.default = { projectId: process.env.PROJECT_ID, + extInstanceId: process.env.EXT_INSTANCE_ID, datasetId: process.env.DATASET_ID, location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map index b66ca980..0fa85fea 100644 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -1 +1 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;CAC1C,CAAC"} \ No newline at end of file +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;CAC1C,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index d2b4c425..c9f1b5ed 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -41,22 +41,34 @@ exports.createBigQueryConnection = functions.tasks const runtime = (0, extensions_1.getExtensions)().runtime(); console.log("Task received => ", task); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const connectionId = `ext-bigquery-dlp-function`; + const connectionId = config_1.default.extInstanceId; + var connection; try { - const connection = await bigqueryConnectionClient.createConnection({ + connection = await bigqueryConnectionClient.createConnection({ parent: parent, connectionId: connectionId, connection: { cloudResource: { - serviceAccountId: `ext-bigquery-dlp-function@${config_1.default.projectId}.iam.gserviceaccount.com`, + serviceAccountId: `${connectionId}@${config_1.default.projectId}.iam.gserviceaccount.com`, }, name: connectionId, - friendlyName: "Deidentify Data", + friendlyName: "DLP Extension", }, }); - functions.logger.info("Connection 1 created => ", connection); - if (connection) { - const query = ` + functions.logger.info("Connection successfully created 🎉", connection); + } + catch (error) { + if (error["code"] === 6) { + functions.logger.info(`Connection ${connectionId} already exists, will continue creating functions`); + } + else { + functions.logger.error(error); + await runtime.setProcessingState("PROCESSING_FAILED", "Error creating connection. Check logs for more details."); + return; + } + } + try { + const query = ` BEGIN CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` @@ -71,18 +83,16 @@ exports.createBigQueryConnection = functions.tasks ); END; `; - const options = { - query: query, - location: config_1.default.location, - }; - // Run the query as a job - const [job] = await bigqueryClient.createQueryJob(options); - functions.logger.debug(`Job ${job.id} started.`); - // Wait for the query to finish - const [rows] = await job.getQueryResults(); - functions.logger.debug("Rows: ", rows); - await runtime.setProcessingState("PROCESSING_COMPLETE", "Connections created successfully."); - } + const options = { + query: query, + location: config_1.default.location, + }; + // Run the query as a job + const [job] = await bigqueryClient.createQueryJob(options); + functions.logger.debug(`Job ${job.id} started.`); + // Wait for the query to finish + await job.getQueryResults(); + await runtime.setProcessingState("PROCESSING_COMPLETE", "Connections created successfully."); } catch (error) { functions.logger.error(error); diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 400c66a8..c1556d9d 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AAEtB,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAAC,KAAK,EAAE,GAAG,CAAC;aACjE,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAAC,KAAK,EAAE,GAAG,CAAC;aAC/D,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,2BAA2B,CAAC;IAEjD,IAAI;QACF,MAAM,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YACjE,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,6BAA6B,gBAAM,CAAC,SAAS,0BAA0B;iBAC1F;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,iBAAiB;aAChC;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,0BAA0B,EAAE,UAAU,CAAC,CAAC;QAE9D,IAAI,UAAU,EAAE;YACd,MAAM,KAAK,GAAG;;8BAEQ,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;YAEH,MAAM,OAAO,GAAG;gBACd,KAAK,EAAE,KAAK;gBACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;aAC1B,CAAC;YAEF,yBAAyB;YACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;YAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;YAEjD,+BAA+B;YAC/B,MAAM,CAAC,IAAI,CAAC,GAAG,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;YAE3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,EAAE,IAAI,CAAC,CAAC;YAEvC,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;SACH;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AAEtB,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAAC,KAAK,EAAE,GAAG,CAAC;aACjE,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAAC,KAAK,EAAE,GAAG,CAAC;aAC/D,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,gBAAM,CAAC,aAAa,CAAC;IAC1C,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 7df3f2a0..95718ec2 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -16,6 +16,7 @@ export default { projectId: process.env.PROJECT_ID, + extInstanceId: process.env.EXT_INSTANCE_ID, datasetId: process.env.DATASET_ID, location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 98c737a7..7a40f77f 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -51,25 +51,41 @@ exports.createBigQueryConnection = functions.tasks console.log("Task received => ", task); const parent = `projects/${config.projectId}/locations/${config.location}`; - const connectionId = `ext-bigquery-dlp-function`; + const connectionId = config.extInstanceId; + var connection; try { - const connection = await bigqueryConnectionClient.createConnection({ + connection = await bigqueryConnectionClient.createConnection({ parent: parent, connectionId: connectionId, connection: { cloudResource: { - serviceAccountId: `ext-bigquery-dlp-function@${config.projectId}.iam.gserviceaccount.com`, + serviceAccountId: `${connectionId}@${config.projectId}.iam.gserviceaccount.com`, }, name: connectionId, - friendlyName: "Deidentify Data", + friendlyName: "DLP Extension", }, }); - functions.logger.info("Connection 1 created => ", connection); + functions.logger.info("Connection successfully created 🎉", connection); + } catch (error: any) { + if (error["code"] === 6) { + functions.logger.info( + `Connection ${connectionId} already exists, will continue creating functions` + ); + } else { + functions.logger.error(error); + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Error creating connection. Check logs for more details." + ); + + return; + } + } - if (connection) { - const query = ` + try { + const query = ` BEGIN CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` @@ -85,25 +101,22 @@ exports.createBigQueryConnection = functions.tasks END; `; - const options = { - query: query, - location: config.location, - }; - - // Run the query as a job - const [job] = await bigqueryClient.createQueryJob(options); - functions.logger.debug(`Job ${job.id} started.`); + const options = { + query: query, + location: config.location, + }; - // Wait for the query to finish - const [rows] = await job.getQueryResults(); + // Run the query as a job + const [job] = await bigqueryClient.createQueryJob(options); + functions.logger.debug(`Job ${job.id} started.`); - functions.logger.debug("Rows: ", rows); + // Wait for the query to finish + await job.getQueryResults(); - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - "Connections created successfully." - ); - } + await runtime.setProcessingState( + "PROCESSING_COMPLETE", + "Connections created successfully." + ); } catch (error) { functions.logger.error(error); From 5d6fb021514194f04533923678a1ac147b2e447e Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 21 Dec 2022 17:53:11 +0300 Subject: [PATCH 12/37] docs: generate readme --- bigquery-dlp-function/README.md | 60 ++++++++++++++++++++ bigquery-dlp-function/functions/package.json | 3 +- 2 files changed, 62 insertions(+), 1 deletion(-) diff --git a/bigquery-dlp-function/README.md b/bigquery-dlp-function/README.md index e69de29b..12735121 100644 --- a/bigquery-dlp-function/README.md +++ b/bigquery-dlp-function/README.md @@ -0,0 +1,60 @@ +# BigQuery DLP Remote Function + +**Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) + +**Description**: TODO + +**Configuration Parameters:** + +- DLP Transformation Method: The method used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. + +- BigQuery Dataset ID: The ID of the dataset where the extension will create a connection. + +- Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). + +**Cloud Functions:** + +- **createBigQueryConnection:** Creates a BigQuery connection. + +- **deidentifyData:** TODO + +**APIs Used**: + +- bigquery.googleapis.com (Reason: Powers all BigQuery tasks performed by the extension.) + +- bigqueryconnection.googleapis.com (Reason: Allows the extension to create a BigQuery connection.) + +- dlp.googleapis.com (Reason: Allows the extension to use DLP services.) + +**Access Required**: + +This extension will operate with the following project IAM roles: + +- bigquery.jobUser (Reason: Allows the extension to create BigQuery jobs.) + +- bigquery.dataOwner (Reason: Allows the extension to create BigQuery routines.) + +- bigquery.connectionAdmin (Reason: Allows the extension to create a BigQuery connection.) + +- dlp.user (Reason: Allows the extension to use DLP services.) + +--- + +## 🧩 Install this experimental extension + +> ⚠️ **Experimental**: This extension is available for testing as an _experimental_ release. It has not been as thoroughly tested as the officially released extensions, and future updates might introduce breaking changes. If you use this extension, please [report bugs and make feature requests](https://github.com/firebase/experimental-extensions/issues/new/choose) in our GitHub repository. + +### Console + +[![Install this extension in your Firebase project](../install-extension.png?raw=true "Install this extension in your Firebase project")](https://console.firebase.google.com/project/_/extensions/install?ref=firebase/bigquery-dlp-function) + +### Firebase CLI + +```bash +firebase ext:install firebase/bigquery-dlp-function --project= +``` + +> Learn more about installing extensions in the Firebase Extensions documentation: [console](https://firebase.google.com/docs/extensions/install-extensions?platform=console), [CLI](https://firebase.google.com/docs/extensions/install-extensions?platform=cli) + +--- + diff --git a/bigquery-dlp-function/functions/package.json b/bigquery-dlp-function/functions/package.json index b2597a10..2e9c4868 100644 --- a/bigquery-dlp-function/functions/package.json +++ b/bigquery-dlp-function/functions/package.json @@ -8,7 +8,8 @@ "shell": "npm run build && firebase functions:shell", "start": "npm run shell", "deploy": "firebase deploy --only functions", - "logs": "firebase functions:log" + "logs": "firebase functions:log", + "generate-readme": "node ../../generate-experimental-readme.js bigquery-dlp-function > ../README.md" }, "main": "lib/index.js", "dependencies": { From 552906e24da9ac503dd3bfebf76ba45f59893d4e Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 21 Dec 2022 17:54:38 +0300 Subject: [PATCH 13/37] - --- bigquery-dlp-function/functions/lib/index.js | 1 - bigquery-dlp-function/functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index c9f1b5ed..1fac7934 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -39,7 +39,6 @@ exports.createBigQueryConnection = functions.tasks .taskQueue() .onDispatch(async (task) => { const runtime = (0, extensions_1.getExtensions)().runtime(); - console.log("Task received => ", task); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; const connectionId = config_1.default.extInstanceId; var connection; diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index c1556d9d..6a31e928 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AAEtB,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAAC,KAAK,EAAE,GAAG,CAAC;aACjE,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAAC,KAAK,EAAE,GAAG,CAAC;aAC/D,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,OAAO,CAAC,GAAG,CAAC,mBAAmB,EAAE,IAAI,CAAC,CAAC;IAEvC,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,gBAAM,CAAC,aAAa,CAAC;IAC1C,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AAEtB,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAAC,KAAK,EAAE,GAAG,CAAC;aACjE,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAAC,KAAK,EAAE,GAAG,CAAC;aAC/D,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,gBAAM,CAAC,aAAa,CAAC;IAC1C,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 7a40f77f..d9893030 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -48,8 +48,6 @@ exports.createBigQueryConnection = functions.tasks .onDispatch(async (task) => { const runtime = getExtensions().runtime(); - console.log("Task received => ", task); - const parent = `projects/${config.projectId}/locations/${config.location}`; const connectionId = config.extInstanceId; var connection; From c4f7c28700f5c20583ada04de177e4686e96ca28 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 21 Dec 2022 18:47:30 +0300 Subject: [PATCH 14/37] feat: transformation types --- .../functions/lib/deidentify.js | 100 +++++++-------- .../functions/lib/deidentify.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 13 +- .../functions/lib/index.js.map | 2 +- .../functions/lib/reidentify.js | 42 +++++++ .../functions/lib/reidentify.js.map | 2 +- .../functions/lib/transofmrations.js | 73 +++++++++++ .../functions/lib/transofmrations.js.map | 1 + .../functions/src/deidentify.ts | 119 +++++++----------- bigquery-dlp-function/functions/src/index.ts | 24 ++-- .../functions/src/reidentify.ts | 56 +++++++++ .../functions/src/transofmrations.ts | 78 ++++++++++++ 12 files changed, 364 insertions(+), 148 deletions(-) create mode 100644 bigquery-dlp-function/functions/lib/transofmrations.js create mode 100644 bigquery-dlp-function/functions/lib/transofmrations.js.map create mode 100644 bigquery-dlp-function/functions/src/transofmrations.ts diff --git a/bigquery-dlp-function/functions/lib/deidentify.js b/bigquery-dlp-function/functions/lib/deidentify.js index e9bef4ae..8b7222a2 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js +++ b/bigquery-dlp-function/functions/lib/deidentify.js @@ -1,8 +1,7 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.deidentifyWithRecordTransformations = exports.deidentifyWithInfoTypeTransformations = void 0; +exports.deidentifyWithInfoTypeTransformations = void 0; const functions = require("firebase-functions"); -const config_1 = require("./config"); // The maximum number of days to shift a date backward const lowerBoundDays = 1; // The maximum number of days to shift a date forward @@ -46,34 +45,16 @@ function rowsToTable(rows) { * * @returns {string} The deidentified text. */ -async function deidentifyWithInfoTypeTransformations(rows, client, mask, numberToMask) { +async function deidentifyWithInfoTypeTransformations(rows, client, transformation) { var _a; const deidentifiedItems = []; - const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const deidentifyConfig = { - parent: parent, - deidentifyConfig: { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", - numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, - }, - }, - }, - ], - }, - }, - }; for (const row of rows) { const data = row[0]; functions.logger.debug(data); for (const key in data) { if (data.hasOwnProperty(key)) { const element = data[key]; - const request = Object.assign(Object.assign({}, deidentifyConfig), { item: { value: element }, parent: parent }); + const request = Object.assign(Object.assign({}, transformation.config), { item: { value: element } }); const [response] = await client.deidentifyContent(request); data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; } @@ -94,40 +75,43 @@ exports.deidentifyWithInfoTypeTransformations = deidentifyWithInfoTypeTransforma * * @returns {string} The deidentified text. */ -async function deidentifyWithRecordTransformations(rows, client, mask, numberToMask) { - var _a, _b; - const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - // Construct DateShiftConfig - const dateShiftConfig = { - lowerBoundDays: lowerBoundDays, - upperBoundDays: upperBoundDays, - }; - // Construct de-identification request - const request = { - parent: parent, - deidentifyConfig: { - recordTransformations: { - fieldTransformations: [ - { - primitiveTransformation: { - dateShiftConfig: dateShiftConfig, - characterMaskConfig: { - maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", - numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, - }, - }, - }, - ], - }, - }, - item: { - table: rowsToTable(rows), - }, - }; - // Run deidentification request - const [response] = await client.deidentifyContent(request); - const tableRows = (_b = (_a = response.item) === null || _a === void 0 ? void 0 : _a.table) === null || _b === void 0 ? void 0 : _b.rows; - return tableRows; -} -exports.deidentifyWithRecordTransformations = deidentifyWithRecordTransformations; +// export async function deidentifyWithRecordTransformations( +// rows: any, +// client: DlpServiceClient, +// mask?: string, +// numberToMask?: number +// ) { +// const parent = `projects/${config.projectId}/locations/${config.location}`; +// // Construct DateShiftConfig +// const dateShiftConfig = { +// lowerBoundDays: lowerBoundDays, +// upperBoundDays: upperBoundDays, +// }; +// // Construct de-identification request +// const request: DeidentifyRequest = { +// parent: parent, +// deidentifyConfig: { +// recordTransformations: { +// fieldTransformations: [ +// { +// primitiveTransformation: { +// dateShiftConfig: dateShiftConfig, +// characterMaskConfig: { +// maskingCharacter: mask ?? "x", +// numberToMask: numberToMask ?? 5, +// }, +// }, +// }, +// ], +// }, +// }, +// item: { +// table: rowsToTable(rows), +// }, +// }; +// // Run deidentification request +// const [response] = await client.deidentifyContent(request); +// const tableRows = response.item?.table?.rows; +// return tableRows; +// } //# sourceMappingURL=deidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js.map b/bigquery-dlp-function/functions/lib/deidentify.js.map index bf6c310b..2c4b2c4e 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js.map +++ b/bigquery-dlp-function/functions/lib/deidentify.js.map @@ -1 +1 @@ -{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,qCAA8B;AAI9B,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,qDAAqD;AACrD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,SAAS,WAAW,CAAC,IAAQ;IAC3B,wDAAwD;IACxD,MAAM,UAAU,GAAG,CAAC,GAAQ,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YAChD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,OAAO;oBACL,SAAS,EAAE;wBACT,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE;wBACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBACpB;iBACF,CAAC;aACH;iBAAM;gBACL,yCAAyC;gBACzC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;aAC1C;QACH,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG;QACZ,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAS,EACT,MAAwB,EACxB,IAAa,EACb,YAAqB;;IAErB,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAsB;QAC1C,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAlDD,sFAkDC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mCAAmC,CACvD,IAAS,EACT,MAAwB,EACxB,IAAa,EACb,YAAqB;;IAErB,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,4BAA4B;IAC5B,MAAM,eAAe,GAAG;QACtB,cAAc,EAAE,cAAc;QAC9B,cAAc,EAAE,cAAc;KAC/B,CAAC;IAEF,sCAAsC;IACtC,MAAM,OAAO,GAAsB;QACjC,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,qBAAqB,EAAE;gBACrB,oBAAoB,EAAE;oBACpB;wBACE,uBAAuB,EAAE;4BACvB,eAAe,EAAE,eAAe;4BAChC,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF;QACD,IAAI,EAAE;YACJ,KAAK,EAAE,WAAW,CAAC,IAAI,CAAC;SACzB;KACF,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAC3D,MAAM,SAAS,GAAG,MAAA,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,0CAAE,IAAI,CAAC;IAE7C,OAAO,SAAS,CAAC;AACnB,CAAC;AA1CD,kFA0CC"} \ No newline at end of file +{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAKhD,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,qDAAqD;AACrD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,SAAS,WAAW,CAAC,IAAQ;IAC3B,wDAAwD;IACxD,MAAM,UAAU,GAAG,CAAC,GAAQ,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YAChD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,OAAO;oBACL,SAAS,EAAE;wBACT,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE;wBACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBACpB;iBACF,CAAC;aACH;iBAAM;gBACL,yCAAyC;gBACzC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;aAC1C;QACH,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG;QACZ,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,MAAM,KACxB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACzB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA7BD,sFA6BC;AAED;;;;;;;;;GASG;AACH,6DAA6D;AAC7D,eAAe;AACf,8BAA8B;AAC9B,mBAAmB;AACnB,0BAA0B;AAC1B,MAAM;AACN,gFAAgF;AAEhF,iCAAiC;AACjC,8BAA8B;AAC9B,sCAAsC;AACtC,sCAAsC;AACtC,OAAO;AAEP,2CAA2C;AAC3C,yCAAyC;AACzC,sBAAsB;AACtB,0BAA0B;AAC1B,iCAAiC;AACjC,kCAAkC;AAClC,cAAc;AACd,yCAAyC;AACzC,kDAAkD;AAClD,uCAAuC;AACvC,iDAAiD;AACjD,mDAAmD;AACnD,mBAAmB;AACnB,iBAAiB;AACjB,eAAe;AACf,aAAa;AACb,WAAW;AACX,SAAS;AACT,cAAc;AACd,kCAAkC;AAClC,SAAS;AACT,OAAO;AAEP,oCAAoC;AACpC,gEAAgE;AAChE,kDAAkD;AAElD,sBAAsB;AACtB,IAAI"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 1fac7934..ebb202ff 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -8,6 +8,7 @@ const bigquery_1 = require("@google-cloud/bigquery"); const extensions_1 = require("firebase-admin/extensions"); const config_1 = require("./config"); const deidentify_1 = require("./deidentify"); +const transofmrations_1 = require("./transofmrations"); admin.initializeApp(); const dlp = new dlp_1.default.DlpServiceClient(); const bigqueryClient = new bigquery_1.BigQuery(); @@ -18,14 +19,14 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => try { if (userDefinedContext.method === "INFO_TYPE") { response.send({ - replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp), - }); - } - else if (userDefinedContext.method === "RECORD") { - response.send({ - replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp), + replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, new transofmrations_1.MaskTransformation()), }); } + // else if (userDefinedContext.method === "RECORD") { + // response.send({ + // replies: await deidentifyWithRecordTransformations(calls, dlp), + // }); + // } else { response.status(400).send("Invalid method"); } diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 6a31e928..1bf0da64 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AAEtB,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAAC,KAAK,EAAE,GAAG,CAAC;aACjE,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAAC,KAAK,EAAE,GAAG,CAAC;aAC/D,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,gBAAM,CAAC,aAAa,CAAC;IAC1C,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAAuD;AAEvD,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,IAAI,oCAAkB,EAAE,CACzB;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,gBAAM,CAAC,aAAa,CAAC;IAC1C,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js b/bigquery-dlp-function/functions/lib/reidentify.js index 0781ad93..d884dbe6 100644 --- a/bigquery-dlp-function/functions/lib/reidentify.js +++ b/bigquery-dlp-function/functions/lib/reidentify.js @@ -1,2 +1,44 @@ "use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.reidentifyWithInfoTypeTransformations = void 0; +const functions = require("firebase-functions"); +const config_1 = require("./config"); +async function reidentifyWithInfoTypeTransformations(rows, client) { + var _a; + const reidentifiedItems = []; + const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + const reidentifyConfig = { + parent: parent, + reidentifyConfig: { + transformationErrorHandling: { + throwError: true, + }, + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: {}, + }, + }, + ], + }, + }, + }; + for (const row of rows) { + const data = row[0]; + functions.logger.debug(data); + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = Object.assign(Object.assign({}, reidentifyConfig), { item: { value: element }, parent: parent }); + const [response] = await client.deidentifyContent(request); + data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; + } + } + functions.logger.debug(data); + reidentifiedItems.push(data); + } + return reidentifiedItems; +} +exports.reidentifyWithInfoTypeTransformations = reidentifyWithInfoTypeTransformations; //# sourceMappingURL=reidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js.map b/bigquery-dlp-function/functions/lib/reidentify.js.map index bcf5dcce..d709f4db 100644 --- a/bigquery-dlp-function/functions/lib/reidentify.js.map +++ b/bigquery-dlp-function/functions/lib/reidentify.js.map @@ -1 +1 @@ -{"version":3,"file":"reidentify.js","sourceRoot":"","sources":["../src/reidentify.ts"],"names":[],"mappings":""} \ No newline at end of file +{"version":3,"file":"reidentify.js","sourceRoot":"","sources":["../src/reidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,qCAA8B;AAIvB,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB;;IAExB,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAsB;QAC1C,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,2BAA2B,EAAE;gBAC3B,UAAU,EAAE,IAAI;aACjB;YACD,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE,EAAE;yBACxB;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAhDD,sFAgDC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js new file mode 100644 index 00000000..f7bbdaa8 --- /dev/null +++ b/bigquery-dlp-function/functions/lib/transofmrations.js @@ -0,0 +1,73 @@ +"use strict"; +Object.defineProperty(exports, "__esModule", { value: true }); +exports.RadactTransformation = exports.MaskTransformation = void 0; +const config_1 = require("./config"); +class Transformation { + constructor() { + this.config = {}; + this.parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; + } +} +class MaskTransformation extends Transformation { + /** + * Replace a value by a mask character. + * + * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. + * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. + */ + constructor(mask, numberToMask) { + super(); + const maskingConfig = { + transformationErrorHandling: { + throwError: true, + }, + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", + numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, + }, + }, + }, + ], + }, + }; + this.config = { + parent: this.parent, + reidentifyConfig: maskingConfig, + deidentifyConfig: maskingConfig, + }; + } +} +exports.MaskTransformation = MaskTransformation; +class RadactTransformation extends Transformation { + /** + * Redacts a value by removing it. + */ + constructor() { + super(); + const maskingConfig = { + transformationErrorHandling: { + throwError: true, + }, + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }; + this.config = { + parent: this.parent, + reidentifyConfig: maskingConfig, + deidentifyConfig: maskingConfig, + }; + } +} +exports.RadactTransformation = RadactTransformation; +//# sourceMappingURL=transofmrations.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js.map b/bigquery-dlp-function/functions/lib/transofmrations.js.map new file mode 100644 index 00000000..ceac4591 --- /dev/null +++ b/bigquery-dlp-function/functions/lib/transofmrations.js.map @@ -0,0 +1 @@ +{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAM9B,MAAM,cAAc;IAIlB;QAFA,WAAM,GAA0C,EAAE,CAAC;QAGjD,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC5E,CAAC;CACF;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC9C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,2BAA2B,EAAE;gBAC3B,UAAU,EAAE,IAAI;aACjB;YACD,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG;YACZ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;YAC/B,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AAjCD,gDAiCC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACtD;;OAEG;IACH;QACE,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,2BAA2B,EAAE;gBAC3B,UAAU,EAAE,IAAI;aACjB;YACD,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG;YACZ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;YAC/B,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AA3BD,oDA2BC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index 0f4fa83b..73f913e5 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -1,9 +1,7 @@ import * as functions from "firebase-functions"; -import { DlpServiceClient, protos } from "@google-cloud/dlp"; +import { DlpServiceClient } from "@google-cloud/dlp"; -import config from "./config"; - -type DeidentifyRequest = protos.google.privacy.dlp.v2.IDeidentifyContentRequest; +import { MaskTransformation, RadactTransformation } from "./transofmrations"; // The maximum number of days to shift a date backward const lowerBoundDays = 1; @@ -53,32 +51,12 @@ function rowsToTable(rows: []) { * @returns {string} The deidentified text. */ export async function deidentifyWithInfoTypeTransformations( - rows: any, + rows: [], client: DlpServiceClient, - mask?: string, - numberToMask?: number + transformation: MaskTransformation | RadactTransformation ) { const deidentifiedItems = []; - const parent = `projects/${config.projectId}/locations/${config.location}`; - const deidentifyConfig: DeidentifyRequest = { - parent: parent, - deidentifyConfig: { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, - }, - }, - }, - ], - }, - }, - }; - for (const row of rows) { const data = row[0] as Record; functions.logger.debug(data); @@ -87,9 +65,8 @@ export async function deidentifyWithInfoTypeTransformations( if (data.hasOwnProperty(key)) { const element = data[key]; const request = { - ...deidentifyConfig, + ...transformation.config, item: { value: element }, - parent: parent, }; const [response] = await client.deidentifyContent(request); @@ -114,46 +91,46 @@ export async function deidentifyWithInfoTypeTransformations( * * @returns {string} The deidentified text. */ -export async function deidentifyWithRecordTransformations( - rows: any, - client: DlpServiceClient, - mask?: string, - numberToMask?: number -) { - const parent = `projects/${config.projectId}/locations/${config.location}`; - - // Construct DateShiftConfig - const dateShiftConfig = { - lowerBoundDays: lowerBoundDays, - upperBoundDays: upperBoundDays, - }; - - // Construct de-identification request - const request: DeidentifyRequest = { - parent: parent, - deidentifyConfig: { - recordTransformations: { - fieldTransformations: [ - { - primitiveTransformation: { - dateShiftConfig: dateShiftConfig, - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, - }, - }, - }, - ], - }, - }, - item: { - table: rowsToTable(rows), - }, - }; - - // Run deidentification request - const [response] = await client.deidentifyContent(request); - const tableRows = response.item?.table?.rows; - - return tableRows; -} +// export async function deidentifyWithRecordTransformations( +// rows: any, +// client: DlpServiceClient, +// mask?: string, +// numberToMask?: number +// ) { +// const parent = `projects/${config.projectId}/locations/${config.location}`; + +// // Construct DateShiftConfig +// const dateShiftConfig = { +// lowerBoundDays: lowerBoundDays, +// upperBoundDays: upperBoundDays, +// }; + +// // Construct de-identification request +// const request: DeidentifyRequest = { +// parent: parent, +// deidentifyConfig: { +// recordTransformations: { +// fieldTransformations: [ +// { +// primitiveTransformation: { +// dateShiftConfig: dateShiftConfig, +// characterMaskConfig: { +// maskingCharacter: mask ?? "x", +// numberToMask: numberToMask ?? 5, +// }, +// }, +// }, +// ], +// }, +// }, +// item: { +// table: rowsToTable(rows), +// }, +// }; + +// // Run deidentification request +// const [response] = await client.deidentifyContent(request); +// const tableRows = response.item?.table?.rows; + +// return tableRows; +// } diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index d9893030..51b11bb5 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -6,10 +6,8 @@ import { BigQuery } from "@google-cloud/bigquery"; import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; -import { - deidentifyWithInfoTypeTransformations, - deidentifyWithRecordTransformations, -} from "./deidentify"; +import { deidentifyWithInfoTypeTransformations } from "./deidentify"; +import { MaskTransformation } from "./transofmrations"; admin.initializeApp(); @@ -26,13 +24,19 @@ exports.deidentifyData = functions.https.onRequest( try { if (userDefinedContext.method === "INFO_TYPE") { response.send({ - replies: await deidentifyWithInfoTypeTransformations(calls, dlp), + replies: await deidentifyWithInfoTypeTransformations( + calls, + dlp, + new MaskTransformation() + ), }); - } else if (userDefinedContext.method === "RECORD") { - response.send({ - replies: await deidentifyWithRecordTransformations(calls, dlp), - }); - } else { + } + // else if (userDefinedContext.method === "RECORD") { + // response.send({ + // replies: await deidentifyWithRecordTransformations(calls, dlp), + // }); + // } + else { response.status(400).send("Invalid method"); } } catch (error) { diff --git a/bigquery-dlp-function/functions/src/reidentify.ts b/bigquery-dlp-function/functions/src/reidentify.ts index e69de29b..502498fe 100644 --- a/bigquery-dlp-function/functions/src/reidentify.ts +++ b/bigquery-dlp-function/functions/src/reidentify.ts @@ -0,0 +1,56 @@ +import * as functions from "firebase-functions"; +import { DlpServiceClient, protos } from "@google-cloud/dlp"; + +import config from "./config"; + +type ReidentifyRequest = protos.google.privacy.dlp.v2.IReidentifyContentRequest; + +export async function reidentifyWithInfoTypeTransformations( + rows: [], + client: DlpServiceClient +) { + const reidentifiedItems = []; + + const parent = `projects/${config.projectId}/locations/${config.location}`; + const reidentifyConfig: ReidentifyRequest = { + parent: parent, + reidentifyConfig: { + transformationErrorHandling: { + throwError: true, + }, + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: {}, + }, + }, + ], + }, + }, + }; + + for (const row of rows) { + const data = row[0] as Record; + functions.logger.debug(data); + + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = { + ...reidentifyConfig, + item: { value: element }, + parent: parent, + }; + + const [response] = await client.deidentifyContent(request); + data[key] = response.item?.value; + } + } + + functions.logger.debug(data); + reidentifiedItems.push(data); + } + + return reidentifiedItems; +} diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts new file mode 100644 index 00000000..fc826163 --- /dev/null +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -0,0 +1,78 @@ +import config from "./config"; +import { protos } from "@google-cloud/dlp"; + +type ReidentifyRequest = protos.google.privacy.dlp.v2.IReidentifyContentRequest; +type DeidentifyRequest = protos.google.privacy.dlp.v2.IDeidentifyContentRequest; + +class Transformation { + parent: string; + config: DeidentifyRequest | ReidentifyRequest = {}; + + constructor() { + this.parent = `projects/${config.projectId}/locations/${config.location}`; + } +} + +export class MaskTransformation extends Transformation { + /** + * Replace a value by a mask character. + * + * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. + * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. + */ + constructor(mask?: string, numberToMask?: number) { + super(); + const maskingConfig = { + transformationErrorHandling: { + throwError: true, + }, + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }; + + this.config = { + parent: this.parent, + reidentifyConfig: maskingConfig, + deidentifyConfig: maskingConfig, + }; + } +} + +export class RadactTransformation extends Transformation { + /** + * Redacts a value by removing it. + */ + constructor() { + super(); + const maskingConfig = { + transformationErrorHandling: { + throwError: true, + }, + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }; + + this.config = { + parent: this.parent, + reidentifyConfig: maskingConfig, + deidentifyConfig: maskingConfig, + }; + } +} From 1cfd5d3ed8ef8f681382eb01f2023deeb10002d3 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Wed, 21 Dec 2022 19:26:42 +0300 Subject: [PATCH 15/37] feat: add technique as param --- bigquery-dlp-function/extension.yaml | 12 +++++++++++- bigquery-dlp-function/functions/lib/config.js | 1 + bigquery-dlp-function/functions/lib/config.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 9 +++++---- bigquery-dlp-function/functions/lib/index.js.map | 2 +- .../functions/lib/transofmrations.js | 6 +++--- bigquery-dlp-function/functions/src/config.ts | 1 + bigquery-dlp-function/functions/src/deidentify.ts | 4 ++-- bigquery-dlp-function/functions/src/index.ts | 9 +++++---- .../functions/src/transofmrations.ts | 2 +- 10 files changed, 31 insertions(+), 17 deletions(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index dd1f4b6e..13295a9a 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -65,7 +65,17 @@ params: # TODO support record type transformations # - label: Record Type Transformations # value: RECORD - required: true + - param: TRANSFORMATION_TECHNIQUE + label: DLP Transformation Technique + description: >- + The technique used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. + type: select + options: + - label: Replace with Masking Character + value: masking + - label: Redact a value (remove it from the data) + value: redact + default: masking - param: DATASET_ID label: BigQuery Dataset ID description: >- diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js index 557073f5..4d019628 100644 --- a/bigquery-dlp-function/functions/lib/config.js +++ b/bigquery-dlp-function/functions/lib/config.js @@ -21,5 +21,6 @@ exports.default = { datasetId: process.env.DATASET_ID, location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, + technique: process.env.TRANSFORMATION_TECHNIQUE, }; //# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map index 0fa85fea..018f76e4 100644 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -1 +1 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;CAC1C,CAAC"} \ No newline at end of file +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;IACzC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;CAChD,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index ebb202ff..45ad33f2 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -38,10 +38,10 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => }); exports.createBigQueryConnection = functions.tasks .taskQueue() - .onDispatch(async (task) => { + .onDispatch(async () => { const runtime = (0, extensions_1.getExtensions)().runtime(); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const connectionId = config_1.default.extInstanceId; + const connectionId = "ext-" + config_1.default.extInstanceId; var connection; try { connection = await bigqueryConnectionClient.createConnection({ @@ -74,12 +74,13 @@ exports.createBigQueryConnection = functions.tasks REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', - user_defined_context = [("method", "${config_1.default.method}")] + user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); END; `; diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 1bf0da64..9e7ae771 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAAuD;AAEvD,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,IAAI,oCAAkB,EAAE,CACzB;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,EAAE,IAAI,EAAE,EAAE;IACzB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,gBAAM,CAAC,aAAa,CAAC;IAC1C,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM;;8BAEjC,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;;;UAG3D,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAAuD;AAEvD,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,IAAI,oCAAkB,EAAE,CACzB;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACnD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js index f7bbdaa8..8ebda16d 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js +++ b/bigquery-dlp-function/functions/lib/transofmrations.js @@ -1,6 +1,6 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.RadactTransformation = exports.MaskTransformation = void 0; +exports.RedactTransformation = exports.MaskTransformation = void 0; const config_1 = require("./config"); class Transformation { constructor() { @@ -42,7 +42,7 @@ class MaskTransformation extends Transformation { } } exports.MaskTransformation = MaskTransformation; -class RadactTransformation extends Transformation { +class RedactTransformation extends Transformation { /** * Redacts a value by removing it. */ @@ -69,5 +69,5 @@ class RadactTransformation extends Transformation { }; } } -exports.RadactTransformation = RadactTransformation; +exports.RedactTransformation = RedactTransformation; //# sourceMappingURL=transofmrations.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 95718ec2..6dad6dd0 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -20,4 +20,5 @@ export default { datasetId: process.env.DATASET_ID, location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, + technique: process.env.TRANSFORMATION_TECHNIQUE, }; diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index 73f913e5..85de1dd9 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -1,7 +1,7 @@ import * as functions from "firebase-functions"; import { DlpServiceClient } from "@google-cloud/dlp"; -import { MaskTransformation, RadactTransformation } from "./transofmrations"; +import { MaskTransformation, RedactTransformation } from "./transofmrations"; // The maximum number of days to shift a date backward const lowerBoundDays = 1; @@ -53,7 +53,7 @@ function rowsToTable(rows: []) { export async function deidentifyWithInfoTypeTransformations( rows: [], client: DlpServiceClient, - transformation: MaskTransformation | RadactTransformation + transformation: MaskTransformation | RedactTransformation ) { const deidentifiedItems = []; diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 51b11bb5..2e57f2fa 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -49,11 +49,11 @@ exports.deidentifyData = functions.https.onRequest( exports.createBigQueryConnection = functions.tasks .taskQueue() - .onDispatch(async (task) => { + .onDispatch(async () => { const runtime = getExtensions().runtime(); const parent = `projects/${config.projectId}/locations/${config.location}`; - const connectionId = config.extInstanceId; + const connectionId = "ext-" + config.extInstanceId; var connection; try { @@ -93,12 +93,13 @@ exports.createBigQueryConnection = functions.tasks REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', - user_defined_context = [("method", "${config.method}")] + user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData' + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); END; `; diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts index fc826163..9407b9b3 100644 --- a/bigquery-dlp-function/functions/src/transofmrations.ts +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -48,7 +48,7 @@ export class MaskTransformation extends Transformation { } } -export class RadactTransformation extends Transformation { +export class RedactTransformation extends Transformation { /** * Redacts a value by removing it. */ From 125a9e1ff6f01a252ab21a14c3f21a8809896d7d Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 13:24:01 +0300 Subject: [PATCH 16/37] fix: reidentify exceptions --- bigquery-dlp-function/README.md | 2 + bigquery-dlp-function/extension.yaml | 7 +++ .../functions/lib/deidentify.js | 2 +- .../functions/lib/deidentify.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 42 ++++++++++++- .../functions/lib/index.js.map | 2 +- .../functions/lib/reidentify.js | 26 +++----- .../functions/lib/reidentify.js.map | 2 +- .../functions/lib/transofmrations.js | 15 ++--- .../functions/lib/transofmrations.js.map | 2 +- .../functions/src/deidentify.ts | 2 +- bigquery-dlp-function/functions/src/index.ts | 59 +++++++++++++++++-- .../functions/src/reidentify.ts | 32 ++++------ .../functions/src/transofmrations.ts | 15 ++--- 14 files changed, 135 insertions(+), 75 deletions(-) diff --git a/bigquery-dlp-function/README.md b/bigquery-dlp-function/README.md index 12735121..087c9bbb 100644 --- a/bigquery-dlp-function/README.md +++ b/bigquery-dlp-function/README.md @@ -8,6 +8,8 @@ - DLP Transformation Method: The method used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. +- DLP Transformation Technique: The technique used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. + - BigQuery Dataset ID: The ID of the dataset where the extension will create a connection. - Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations). diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 13295a9a..c5ddf375 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -52,6 +52,13 @@ resources: location: ${param:LOCATION} runtime: nodejs14 httpsTrigger: {} + - name: reidentifyData + type: firebaseextensions.v1beta.function + description: TODO + properties: + location: ${param:LOCATION} + runtime: nodejs14 + httpsTrigger: {} params: - param: TRANSFORMATION_METHOD diff --git a/bigquery-dlp-function/functions/lib/deidentify.js b/bigquery-dlp-function/functions/lib/deidentify.js index 8b7222a2..94bfa7d0 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js +++ b/bigquery-dlp-function/functions/lib/deidentify.js @@ -54,7 +54,7 @@ async function deidentifyWithInfoTypeTransformations(rows, client, transformatio for (const key in data) { if (data.hasOwnProperty(key)) { const element = data[key]; - const request = Object.assign(Object.assign({}, transformation.config), { item: { value: element } }); + const request = Object.assign(Object.assign({}, transformation.deidentifyConfig), { item: { value: element } }); const [response] = await client.deidentifyContent(request); data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; } diff --git a/bigquery-dlp-function/functions/lib/deidentify.js.map b/bigquery-dlp-function/functions/lib/deidentify.js.map index 2c4b2c4e..b7ca7ada 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js.map +++ b/bigquery-dlp-function/functions/lib/deidentify.js.map @@ -1 +1 @@ -{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAKhD,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,qDAAqD;AACrD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,SAAS,WAAW,CAAC,IAAQ;IAC3B,wDAAwD;IACxD,MAAM,UAAU,GAAG,CAAC,GAAQ,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YAChD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,OAAO;oBACL,SAAS,EAAE;wBACT,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE;wBACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBACpB;iBACF,CAAC;aACH;iBAAM;gBACL,yCAAyC;gBACzC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;aAC1C;QACH,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG;QACZ,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,MAAM,KACxB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACzB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA7BD,sFA6BC;AAED;;;;;;;;;GASG;AACH,6DAA6D;AAC7D,eAAe;AACf,8BAA8B;AAC9B,mBAAmB;AACnB,0BAA0B;AAC1B,MAAM;AACN,gFAAgF;AAEhF,iCAAiC;AACjC,8BAA8B;AAC9B,sCAAsC;AACtC,sCAAsC;AACtC,OAAO;AAEP,2CAA2C;AAC3C,yCAAyC;AACzC,sBAAsB;AACtB,0BAA0B;AAC1B,iCAAiC;AACjC,kCAAkC;AAClC,cAAc;AACd,yCAAyC;AACzC,kDAAkD;AAClD,uCAAuC;AACvC,iDAAiD;AACjD,mDAAmD;AACnD,mBAAmB;AACnB,iBAAiB;AACjB,eAAe;AACf,aAAa;AACb,WAAW;AACX,SAAS;AACT,cAAc;AACd,kCAAkC;AAClC,SAAS;AACT,OAAO;AAEP,oCAAoC;AACpC,gEAAgE;AAChE,kDAAkD;AAElD,sBAAsB;AACtB,IAAI"} \ No newline at end of file +{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAKhD,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,qDAAqD;AACrD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,SAAS,WAAW,CAAC,IAAQ;IAC3B,wDAAwD;IACxD,MAAM,UAAU,GAAG,CAAC,GAAQ,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YAChD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,OAAO;oBACL,SAAS,EAAE;wBACT,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE;wBACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBACpB;iBACF,CAAC;aACH;iBAAM;gBACL,yCAAyC;gBACzC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;aAC1C;QACH,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG;QACZ,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACzB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA7BD,sFA6BC;AAED;;;;;;;;;GASG;AACH,6DAA6D;AAC7D,eAAe;AACf,8BAA8B;AAC9B,mBAAmB;AACnB,0BAA0B;AAC1B,MAAM;AACN,gFAAgF;AAEhF,iCAAiC;AACjC,8BAA8B;AAC9B,sCAAsC;AACtC,sCAAsC;AACtC,OAAO;AAEP,2CAA2C;AAC3C,yCAAyC;AACzC,sBAAsB;AACtB,0BAA0B;AAC1B,iCAAiC;AACjC,kCAAkC;AAClC,cAAc;AACd,yCAAyC;AACzC,kDAAkD;AAClD,uCAAuC;AACvC,iDAAiD;AACjD,mDAAmD;AACnD,mBAAmB;AACnB,iBAAiB;AACjB,eAAe;AACf,aAAa;AACb,WAAW;AACX,SAAS;AACT,cAAc;AACd,kCAAkC;AAClC,SAAS;AACT,OAAO;AAEP,oCAAoC;AACpC,gEAAgE;AAChE,kDAAkD;AAElD,sBAAsB;AACtB,IAAI"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 45ad33f2..7bde8eee 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -9,6 +9,7 @@ const extensions_1 = require("firebase-admin/extensions"); const config_1 = require("./config"); const deidentify_1 = require("./deidentify"); const transofmrations_1 = require("./transofmrations"); +const reidentify_1 = require("./reidentify"); admin.initializeApp(); const dlp = new dlp_1.default.DlpServiceClient(); const bigqueryClient = new bigquery_1.BigQuery(); @@ -18,8 +19,45 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => functions.logger.debug("Incoming request from BigQuery", calls); try { if (userDefinedContext.method === "INFO_TYPE") { + var transformation; + switch (config_1.default.technique) { + case "redact": + transformation = new transofmrations_1.RedactTransformation(); + break; + default: + transformation = new transofmrations_1.MaskTransformation(); + } response.send({ - replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, new transofmrations_1.MaskTransformation()), + replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, transformation), + }); + } + // else if (userDefinedContext.method === "RECORD") { + // response.send({ + // replies: await deidentifyWithRecordTransformations(calls, dlp), + // }); + // } + else { + response.status(400).send("Invalid method"); + } + } + catch (error) { + functions.logger.error(error); + response.status(500).send(`errorMessage: ${error}`); + } +}); +exports.reidentifyData = functions.https.onRequest(async (request, response) => { + const { calls, userDefinedContext } = request.body; + functions.logger.debug("Incoming request from BigQuery", calls); + try { + if (userDefinedContext.method === "INFO_TYPE") { + var transformation; + switch (config_1.default.technique) { + default: + response.status(400).send("Invalid or irreversable technique"); + return; + } + response.send({ + replies: await (0, reidentify_1.reidentifyWithInfoTypeTransformations)(calls, dlp, transformation), }); } // else if (userDefinedContext.method === "RECORD") { @@ -79,7 +117,7 @@ exports.createBigQueryConnection = functions.tasks CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-reidentifyData', user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); END; diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 9e7ae771..1e41a633 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAAuD;AAEvD,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,IAAI,oCAAkB,EAAE,CACzB;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACnD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB,KAAK,QAAQ;oBACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;oBAC5C,MAAM;gBAER;oBACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;aAC7C;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB;oBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/D,OAAO;aACV;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACnD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js b/bigquery-dlp-function/functions/lib/reidentify.js index d884dbe6..51491a10 100644 --- a/bigquery-dlp-function/functions/lib/reidentify.js +++ b/bigquery-dlp-function/functions/lib/reidentify.js @@ -2,35 +2,23 @@ Object.defineProperty(exports, "__esModule", { value: true }); exports.reidentifyWithInfoTypeTransformations = void 0; const functions = require("firebase-functions"); +const transofmrations_1 = require("./transofmrations"); const config_1 = require("./config"); -async function reidentifyWithInfoTypeTransformations(rows, client) { +async function reidentifyWithInfoTypeTransformations(rows, client, transformation) { var _a; const reidentifiedItems = []; const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const reidentifyConfig = { - parent: parent, - reidentifyConfig: { - transformationErrorHandling: { - throwError: true, - }, - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: {}, - }, - }, - ], - }, - }, - }; + if (transformation instanceof transofmrations_1.MaskTransformation) { + functions.logger.debug("Mask Transformation is irreversable"); + throw new Error("Mask Transformation is irreversable"); + } for (const row of rows) { const data = row[0]; functions.logger.debug(data); for (const key in data) { if (data.hasOwnProperty(key)) { const element = data[key]; - const request = Object.assign(Object.assign({}, reidentifyConfig), { item: { value: element }, parent: parent }); + const request = Object.assign(Object.assign({}, transformation.reidentifyConfig), { item: { value: element }, parent: parent }); const [response] = await client.deidentifyContent(request); data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; } diff --git a/bigquery-dlp-function/functions/lib/reidentify.js.map b/bigquery-dlp-function/functions/lib/reidentify.js.map index d709f4db..815cbddf 100644 --- a/bigquery-dlp-function/functions/lib/reidentify.js.map +++ b/bigquery-dlp-function/functions/lib/reidentify.js.map @@ -1 +1 @@ -{"version":3,"file":"reidentify.js","sourceRoot":"","sources":["../src/reidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,qCAA8B;AAIvB,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB;;IAExB,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,gBAAgB,GAAsB;QAC1C,MAAM,EAAE,MAAM;QACd,gBAAgB,EAAE;YAChB,2BAA2B,EAAE;gBAC3B,UAAU,EAAE,IAAI;aACjB;YACD,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE,EAAE;yBACxB;qBACF;iBACF;aACF;SACF;KACF,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,gBAAgB,KACnB,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AAhDD,sFAgDC"} \ No newline at end of file +{"version":3,"file":"reidentify.js","sourceRoot":"","sources":["../src/reidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,uDAA6E;AAC7E,qCAA8B;AAEvB,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,IAAI,cAAc,YAAY,oCAAkB,EAAE;QAChD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;KACxD;IAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AArCD,sFAqCC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js index 8ebda16d..a26393ce 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js +++ b/bigquery-dlp-function/functions/lib/transofmrations.js @@ -4,7 +4,8 @@ exports.RedactTransformation = exports.MaskTransformation = void 0; const config_1 = require("./config"); class Transformation { constructor() { - this.config = {}; + this.deidentifyConfig = {}; + this.reidentifyConfig = {}; this.parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; } } @@ -18,9 +19,6 @@ class MaskTransformation extends Transformation { constructor(mask, numberToMask) { super(); const maskingConfig = { - transformationErrorHandling: { - throwError: true, - }, infoTypeTransformations: { transformations: [ { @@ -34,9 +32,8 @@ class MaskTransformation extends Transformation { ], }, }; - this.config = { + this.deidentifyConfig = { parent: this.parent, - reidentifyConfig: maskingConfig, deidentifyConfig: maskingConfig, }; } @@ -49,9 +46,6 @@ class RedactTransformation extends Transformation { constructor() { super(); const maskingConfig = { - transformationErrorHandling: { - throwError: true, - }, infoTypeTransformations: { transformations: [ { @@ -62,9 +56,8 @@ class RedactTransformation extends Transformation { ], }, }; - this.config = { + this.deidentifyConfig = { parent: this.parent, - reidentifyConfig: maskingConfig, deidentifyConfig: maskingConfig, }; } diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js.map b/bigquery-dlp-function/functions/lib/transofmrations.js.map index ceac4591..9a4d02ef 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js.map +++ b/bigquery-dlp-function/functions/lib/transofmrations.js.map @@ -1 +1 @@ -{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAM9B,MAAM,cAAc;IAIlB;QAFA,WAAM,GAA0C,EAAE,CAAC;QAGjD,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC5E,CAAC;CACF;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC9C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,2BAA2B,EAAE;gBAC3B,UAAU,EAAE,IAAI;aACjB;YACD,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG;YACZ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;YAC/B,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AAjCD,gDAiCC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACtD;;OAEG;IACH;QACE,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,2BAA2B,EAAE;gBAC3B,UAAU,EAAE,IAAI;aACjB;YACD,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,MAAM,GAAG;YACZ,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;YAC/B,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AA3BD,oDA2BC"} \ No newline at end of file +{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAM9B,MAAM,cAAc;IAKlB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGvC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC5E,CAAC;CACF;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC9C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AA7BD,gDA6BC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACtD;;OAEG;IACH;QACE,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AAvBD,oDAuBC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index 85de1dd9..95acae5f 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -65,7 +65,7 @@ export async function deidentifyWithInfoTypeTransformations( if (data.hasOwnProperty(key)) { const element = data[key]; const request = { - ...transformation.config, + ...transformation.deidentifyConfig, item: { value: element }, }; diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 2e57f2fa..7f3f65ea 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -7,7 +7,8 @@ import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; import { deidentifyWithInfoTypeTransformations } from "./deidentify"; -import { MaskTransformation } from "./transofmrations"; +import { MaskTransformation, RedactTransformation } from "./transofmrations"; +import { reidentifyWithInfoTypeTransformations } from "./reidentify"; admin.initializeApp(); @@ -23,11 +24,61 @@ exports.deidentifyData = functions.https.onRequest( try { if (userDefinedContext.method === "INFO_TYPE") { + var transformation; + + switch (config.technique) { + case "redact": + transformation = new RedactTransformation(); + break; + + default: + transformation = new MaskTransformation(); + } response.send({ replies: await deidentifyWithInfoTypeTransformations( calls, dlp, - new MaskTransformation() + transformation + ), + }); + } + // else if (userDefinedContext.method === "RECORD") { + // response.send({ + // replies: await deidentifyWithRecordTransformations(calls, dlp), + // }); + // } + else { + response.status(400).send("Invalid method"); + } + } catch (error) { + functions.logger.error(error); + + response.status(500).send(`errorMessage: ${error}`); + } + } +); + +exports.reidentifyData = functions.https.onRequest( + async (request, response) => { + const { calls, userDefinedContext } = request.body; + + functions.logger.debug("Incoming request from BigQuery", calls); + + try { + if (userDefinedContext.method === "INFO_TYPE") { + var transformation; + + switch (config.technique) { + default: + response.status(400).send("Invalid or irreversable technique"); + return; + } + + response.send({ + replies: await reidentifyWithInfoTypeTransformations( + calls, + dlp, + transformation ), }); } @@ -95,10 +146,10 @@ exports.createBigQueryConnection = functions.tasks endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); - CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reindetify(data JSON) RETURNS JSON + CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-reidentifyData', user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); END; diff --git a/bigquery-dlp-function/functions/src/reidentify.ts b/bigquery-dlp-function/functions/src/reidentify.ts index 502498fe..d362ac15 100644 --- a/bigquery-dlp-function/functions/src/reidentify.ts +++ b/bigquery-dlp-function/functions/src/reidentify.ts @@ -1,34 +1,22 @@ import * as functions from "firebase-functions"; -import { DlpServiceClient, protos } from "@google-cloud/dlp"; +import { DlpServiceClient } from "@google-cloud/dlp"; +import { MaskTransformation, RedactTransformation } from "./transofmrations"; import config from "./config"; -type ReidentifyRequest = protos.google.privacy.dlp.v2.IReidentifyContentRequest; - export async function reidentifyWithInfoTypeTransformations( rows: [], - client: DlpServiceClient + client: DlpServiceClient, + transformation: MaskTransformation | RedactTransformation ) { const reidentifiedItems = []; const parent = `projects/${config.projectId}/locations/${config.location}`; - const reidentifyConfig: ReidentifyRequest = { - parent: parent, - reidentifyConfig: { - transformationErrorHandling: { - throwError: true, - }, - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: {}, - }, - }, - ], - }, - }, - }; + + if (transformation instanceof MaskTransformation) { + functions.logger.debug("Mask Transformation is irreversable"); + throw new Error("Mask Transformation is irreversable"); + } for (const row of rows) { const data = row[0] as Record; @@ -38,7 +26,7 @@ export async function reidentifyWithInfoTypeTransformations( if (data.hasOwnProperty(key)) { const element = data[key]; const request = { - ...reidentifyConfig, + ...transformation.reidentifyConfig, item: { value: element }, parent: parent, }; diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts index 9407b9b3..87ebb610 100644 --- a/bigquery-dlp-function/functions/src/transofmrations.ts +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -6,7 +6,8 @@ type DeidentifyRequest = protos.google.privacy.dlp.v2.IDeidentifyContentRequest; class Transformation { parent: string; - config: DeidentifyRequest | ReidentifyRequest = {}; + deidentifyConfig: DeidentifyRequest = {}; + reidentifyConfig: ReidentifyRequest = {}; constructor() { this.parent = `projects/${config.projectId}/locations/${config.location}`; @@ -23,9 +24,6 @@ export class MaskTransformation extends Transformation { constructor(mask?: string, numberToMask?: number) { super(); const maskingConfig = { - transformationErrorHandling: { - throwError: true, - }, infoTypeTransformations: { transformations: [ { @@ -40,9 +38,8 @@ export class MaskTransformation extends Transformation { }, }; - this.config = { + this.deidentifyConfig = { parent: this.parent, - reidentifyConfig: maskingConfig, deidentifyConfig: maskingConfig, }; } @@ -55,9 +52,6 @@ export class RedactTransformation extends Transformation { constructor() { super(); const maskingConfig = { - transformationErrorHandling: { - throwError: true, - }, infoTypeTransformations: { transformations: [ { @@ -69,9 +63,8 @@ export class RedactTransformation extends Transformation { }, }; - this.config = { + this.deidentifyConfig = { parent: this.parent, - reidentifyConfig: maskingConfig, deidentifyConfig: maskingConfig, }; } From 07f51b2fa5b239e16f05be2ba8b76f7a0255537a Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 13:32:39 +0300 Subject: [PATCH 17/37] docs: preinstall --- bigquery-dlp-function/PREINSTALL.md | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/bigquery-dlp-function/PREINSTALL.md b/bigquery-dlp-function/PREINSTALL.md index e69de29b..331979f6 100644 --- a/bigquery-dlp-function/PREINSTALL.md +++ b/bigquery-dlp-function/PREINSTALL.md @@ -0,0 +1,25 @@ +Use this extension to de-identify sensitive data in BigQuery using the [Data Loss Prevention API](https://cloud.google.com/dlp/docs/). + +This extension deploys 2 BigQuery remote functions, this extension: + +- Perform de-identifaction on sensitive data passed as JSON from BigQuery. +- Re-identify sensitive data that were de-identified with reversable techniques. + +You specify the desired DLP technique. All techniques are powered by the Google [Data Loss Prevention API](https://cloud.google.com/dlp/docs/transformations-reference). The options offered are: + +- Replace with Masking. +- Redact a value (remove it from the data). + +#### Additional setup + +Before installing this extension, make sure that you've set up a BigQuery [dataset](https://cloud.google.com/bigquery/docs/datasets) and [table](https://cloud.google.com/bigquery/docs/tables). + +#### Billing + +This extension uses other Firebase or Google Cloud Platform services which may have associated charges: + +- Cloud Data Loss Prevention API +- BigQuery +- Cloud Functions + +When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) From 6eb5dde85ec3352df205b7bfbf4a7692c79e7278 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 13:36:16 +0300 Subject: [PATCH 18/37] docs: postinstall --- bigquery-dlp-function/POSTINSTALL.md | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/bigquery-dlp-function/POSTINSTALL.md b/bigquery-dlp-function/POSTINSTALL.md index e69de29b..aabf6822 100644 --- a/bigquery-dlp-function/POSTINSTALL.md +++ b/bigquery-dlp-function/POSTINSTALL.md @@ -0,0 +1,28 @@ +### See it in action + +1. Go to your project's [BigQuery](https://console.cloud.google.com/bigquery?cloudshell=false&project=${param:PROJECT_ID}) in the Google Cloud console. +2. If it doesn't exist already, create a dataset called `${param:DATASET_ID}`. +3. Create a table that contains the data you want to de-identify. +4. Run the following query to de-identify the data in the table: + +```sql +SELECT + val, + `dev-extensions-testing.bq_testing`.deindetify(TO_JSON(val)) +FROM + `dev-extensions-testing.bq_testing.users` AS val +``` + +5. Run the following query to re-identify the data in the table: + +```sql +SELECT + val, + `dev-extensions-testing.bq_testing`.reindetify(TO_JSON(val)) +FROM + `dev-extensions-testing.bq_testing.users` AS val +``` + +### Monitoring + +As a best practice, you can [monitor the activity](https://firebase.google.com/docs/extensions/manage-installed-extensions#monitor) of your installed extension, including checks on its health, usage, and logs. From f581cc8d045a964528edb619b4cdc40b7e3cd2b6 Mon Sep 17 00:00:00 2001 From: dackers86 Date: Fri, 23 Dec 2022 10:42:48 +0000 Subject: [PATCH 19/37] chore: added changelog --- bigquery-dlp-function/CHANGELOG.md | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 bigquery-dlp-function/CHANGELOG.md diff --git a/bigquery-dlp-function/CHANGELOG.md b/bigquery-dlp-function/CHANGELOG.md new file mode 100644 index 00000000..e27b78db --- /dev/null +++ b/bigquery-dlp-function/CHANGELOG.md @@ -0,0 +1,3 @@ +## Version 0.0.1 + +Alpha release allowing deidentify/reidentify with documentation. From b3475d22b94480732c8f59c04f9bc8f72c4e4c79 Mon Sep 17 00:00:00 2001 From: dackers86 Date: Fri, 23 Dec 2022 10:50:25 +0000 Subject: [PATCH 20/37] chore: renamed to functions --- _emulator/firebase.json | 8 ++------ bigquery-dlp-function/README.md | 4 ++-- bigquery-dlp-function/extension.yaml | 2 +- bigquery-dlp-function/functions/lib/index.js | 4 ++-- bigquery-dlp-function/functions/package.json | 2 +- bigquery-dlp-function/functions/src/index.ts | 4 ++-- 6 files changed, 10 insertions(+), 14 deletions(-) diff --git a/_emulator/firebase.json b/_emulator/firebase.json index 43b8a726..597bab84 100644 --- a/_emulator/firebase.json +++ b/_emulator/firebase.json @@ -2,7 +2,7 @@ "extensions": { "firestore-record-user-acknowledgements": "../firestore-record-user-acknowledgements", "firestore-bundle-server": "../firestore-bundle-server", - "bigquery-dlp-function": "../bigquery-dlp-function" + "bigquery-dlp-functions": "../bigquery-dlp-functions" }, "storage": { "rules": "storage.rules" @@ -36,11 +36,7 @@ }, "hosting": { "public": "dist", - "ignore": [ - "firebase.json", - "**/.*", - "**/node_modules/**" - ], + "ignore": ["firebase.json", "**/.*", "**/node_modules/**"], "rewrites": [ { "source": "/bundles/*", diff --git a/bigquery-dlp-function/README.md b/bigquery-dlp-function/README.md index 087c9bbb..89030289 100644 --- a/bigquery-dlp-function/README.md +++ b/bigquery-dlp-function/README.md @@ -48,12 +48,12 @@ This extension will operate with the following project IAM roles: ### Console -[![Install this extension in your Firebase project](../install-extension.png?raw=true "Install this extension in your Firebase project")](https://console.firebase.google.com/project/_/extensions/install?ref=firebase/bigquery-dlp-function) +[![Install this extension in your Firebase project](../install-extension.png?raw=true "Install this extension in your Firebase project")](https://console.firebase.google.com/project/_/extensions/install?ref=firebase/bigquery-dlp-functions) ### Firebase CLI ```bash -firebase ext:install firebase/bigquery-dlp-function --project= +firebase ext:install firebase/bigquery-dlp-functions --project= ``` > Learn more about installing extensions in the Firebase Extensions documentation: [console](https://firebase.google.com/docs/extensions/install-extensions?platform=console), [CLI](https://firebase.google.com/docs/extensions/install-extensions?platform=cli) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index c5ddf375..64536287 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -1,7 +1,7 @@ # Learn detailed information about the fields of an extension.yaml file in the docs: # https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml -name: bigquery-dlp-function # Identifier for your extension +name: bigquery-dlp-functions # Identifier for your extension version: 0.0.1 # Follow semver versioning specVersion: v1beta # Version of the Firebase Extensions specification diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 7bde8eee..cd78c727 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -111,13 +111,13 @@ exports.createBigQueryConnection = functions.tasks CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-deidentifyData', user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-reidentifyData', + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-reidentifyData', user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); END; diff --git a/bigquery-dlp-function/functions/package.json b/bigquery-dlp-function/functions/package.json index 2e9c4868..02528045 100644 --- a/bigquery-dlp-function/functions/package.json +++ b/bigquery-dlp-function/functions/package.json @@ -9,7 +9,7 @@ "start": "npm run shell", "deploy": "firebase deploy --only functions", "logs": "firebase functions:log", - "generate-readme": "node ../../generate-experimental-readme.js bigquery-dlp-function > ../README.md" + "generate-readme": "node ../../generate-experimental-readme.js bigquery-dlp-functions > ../README.md" }, "main": "lib/index.js", "dependencies": { diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 7f3f65ea..01fdc743 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -143,13 +143,13 @@ exports.createBigQueryConnection = functions.tasks CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-deidentifyData', + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-deidentifyData', user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-function-reidentifyData', + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-reidentifyData', user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); END; From a4fbe899b68026ac2e44dd05024a5d10f6f4e0b4 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 13:58:13 +0300 Subject: [PATCH 21/37] fix: remove default value in DATASET_ID param --- bigquery-dlp-function/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 64536287..10704cd7 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -72,6 +72,7 @@ params: # TODO support record type transformations # - label: Record Type Transformations # value: RECORD + default: INFO_TYPE - param: TRANSFORMATION_TECHNIQUE label: DLP Transformation Technique description: >- @@ -88,7 +89,6 @@ params: description: >- The ID of the dataset where the extension will create a connection. type: string - default: deidentified_data required: true immutable: true - param: LOCATION From fc72158365f4ba7eb5180dcbce63ea31037e6365 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 13:59:45 +0300 Subject: [PATCH 22/37] fix: build --- bigquery-dlp-function/functions/lib/index.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index cd78c727..27743257 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -114,7 +114,7 @@ exports.createBigQueryConnection = functions.tasks endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-deidentifyData', user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reindetify(data JSON) RETURNS JSON + CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` OPTIONS ( endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-reidentifyData', From 081a2a8b242f45e1b2f694bda0bc49fe1c23469a Mon Sep 17 00:00:00 2001 From: dackers86 Date: Fri, 23 Dec 2022 11:15:48 +0000 Subject: [PATCH 23/37] chore: updated changelog before publish --- bigquery-dlp-function/CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/bigquery-dlp-function/CHANGELOG.md b/bigquery-dlp-function/CHANGELOG.md index e27b78db..6f0a5d30 100644 --- a/bigquery-dlp-function/CHANGELOG.md +++ b/bigquery-dlp-function/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.0.2 + +fix: remove default value in DATASET_ID param + ## Version 0.0.1 Alpha release allowing deidentify/reidentify with documentation. From abbfe4b3d8289579432fef1ccf8acb6b9df3bbf2 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 14:17:39 +0300 Subject: [PATCH 24/37] fix: identation --- bigquery-dlp-function/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 10704cd7..058d4f10 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -72,7 +72,7 @@ params: # TODO support record type transformations # - label: Record Type Transformations # value: RECORD - default: INFO_TYPE + default: INFO_TYPE - param: TRANSFORMATION_TECHNIQUE label: DLP Transformation Technique description: >- From f413f6d5c32a4b55dd08b01face87c58e4cf6c22 Mon Sep 17 00:00:00 2001 From: dackers86 Date: Fri, 23 Dec 2022 11:19:53 +0000 Subject: [PATCH 25/37] chore: ext version bump --- bigquery-dlp-function/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 058d4f10..831306bf 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -2,7 +2,7 @@ # https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml name: bigquery-dlp-functions # Identifier for your extension -version: 0.0.1 # Follow semver versioning +version: 0.0.2 # Follow semver versioning specVersion: v1beta # Version of the Firebase Extensions specification author: From fb45b820a733807f5553cbbb880de459ea6c57cf Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 14:22:14 +0300 Subject: [PATCH 26/37] fix: default value --- bigquery-dlp-function/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 831306bf..66ce695e 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -72,7 +72,7 @@ params: # TODO support record type transformations # - label: Record Type Transformations # value: RECORD - default: INFO_TYPE + default: INFO_TYPE - param: TRANSFORMATION_TECHNIQUE label: DLP Transformation Technique description: >- From e797b90d80ceaf62540c6f7977da7c4a87525742 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 14:44:49 +0300 Subject: [PATCH 27/37] fix: instance id --- bigquery-dlp-function/functions/lib/index.js | 18 +++++++++--------- .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 18 +++++++++--------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 27743257..e0908228 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -79,17 +79,17 @@ exports.createBigQueryConnection = functions.tasks .onDispatch(async () => { const runtime = (0, extensions_1.getExtensions)().runtime(); const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const connectionId = "ext-" + config_1.default.extInstanceId; + const instanceId = "ext-" + config_1.default.extInstanceId; var connection; try { connection = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: connectionId, + connectionId: instanceId, connection: { cloudResource: { - serviceAccountId: `${connectionId}@${config_1.default.projectId}.iam.gserviceaccount.com`, + serviceAccountId: `${instanceId}@${config_1.default.projectId}.iam.gserviceaccount.com`, }, - name: connectionId, + name: instanceId, friendlyName: "DLP Extension", }, }); @@ -97,7 +97,7 @@ exports.createBigQueryConnection = functions.tasks } catch (error) { if (error["code"] === 6) { - functions.logger.info(`Connection ${connectionId} already exists, will continue creating functions`); + functions.logger.info(`Connection ${instanceId} already exists, will continue creating functions`); } else { functions.logger.error(error); @@ -109,15 +109,15 @@ exports.createBigQueryConnection = functions.tasks const query = ` BEGIN CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-deidentifyData', + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-deidentifyData', user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${connectionId}\` + REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-reidentifyData', + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-reidentifyData', user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] ); END; diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 1e41a633..fb91bc7f 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB,KAAK,QAAQ;oBACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;oBAC5C,MAAM;gBAER;oBACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;aAC7C;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB;oBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/D,OAAO;aACV;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,YAAY,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACnD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,YAAY;YAC1B,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,YAAY,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAChF;gBACD,IAAI,EAAE,YAAY;gBAClB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,YAAY,mDAAmD,CAC9E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,YAAY;;kCAEtD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS;kDACnB,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB,KAAK,QAAQ;oBACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;oBAC5C,MAAM;gBAER;oBACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;aAC7C;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB;oBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/D,OAAO;aACV;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC9E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,UAAU,mDAAmD,CAC5E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 01fdc743..45a6b925 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -104,18 +104,18 @@ exports.createBigQueryConnection = functions.tasks const runtime = getExtensions().runtime(); const parent = `projects/${config.projectId}/locations/${config.location}`; - const connectionId = "ext-" + config.extInstanceId; + const instanceId = "ext-" + config.extInstanceId; var connection; try { connection = await bigqueryConnectionClient.createConnection({ parent: parent, - connectionId: connectionId, + connectionId: instanceId, connection: { cloudResource: { - serviceAccountId: `${connectionId}@${config.projectId}.iam.gserviceaccount.com`, + serviceAccountId: `${instanceId}@${config.projectId}.iam.gserviceaccount.com`, }, - name: connectionId, + name: instanceId, friendlyName: "DLP Extension", }, }); @@ -124,7 +124,7 @@ exports.createBigQueryConnection = functions.tasks } catch (error: any) { if (error["code"] === 6) { functions.logger.info( - `Connection ${connectionId} already exists, will continue creating functions` + `Connection ${instanceId} already exists, will continue creating functions` ); } else { functions.logger.error(error); @@ -141,15 +141,15 @@ exports.createBigQueryConnection = functions.tasks const query = ` BEGIN CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-deidentifyData', + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/${instanceId}-deidentifyData', user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${connectionId}\` + REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/ext-bigquery-dlp-functions-reidentifyData', + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/${instanceId}-reidentifyData', user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] ); END; From 9f486302768be6957382608c1b370fb156b9eb07 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Fri, 23 Dec 2022 22:19:26 +0300 Subject: [PATCH 28/37] fix: warn when remote functions exists --- bigquery-dlp-function/README.md | 68 +++++++++++++------ bigquery-dlp-function/functions/lib/index.js | 12 +++- .../functions/lib/index.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 20 ++++-- 4 files changed, 71 insertions(+), 31 deletions(-) diff --git a/bigquery-dlp-function/README.md b/bigquery-dlp-function/README.md index 89030289..13795b48 100644 --- a/bigquery-dlp-function/README.md +++ b/bigquery-dlp-function/README.md @@ -4,6 +4,52 @@ **Description**: TODO +--- + +## 🧩 Install this experimental extension + +> ⚠️ **Experimental**: This extension is available for testing as an _experimental_ release. It has not been as thoroughly tested as the officially released extensions, and future updates might introduce breaking changes. If you use this extension, please [report bugs and make feature requests](https://github.com/firebase/experimental-extensions/issues/new/choose) in our GitHub repository. + +### Console + +[![Install this extension in your Firebase project](../install-extension.png?raw=true "Install this extension in your Firebase project")](https://console.firebase.google.com/project/_/extensions/install?ref=firebase/bigquery-dlp-functions) + +### Firebase CLI + +```bash +firebase ext:install firebase/bigquery-dlp-functions --project= +``` + +> Learn more about installing extensions in the Firebase Extensions documentation: [console](https://firebase.google.com/docs/extensions/install-extensions?platform=console), [CLI](https://firebase.google.com/docs/extensions/install-extensions?platform=cli) + +--- + +**Details**: Use this extension to de-identify sensitive data in BigQuery using the [Data Loss Prevention API](https://cloud.google.com/dlp/docs/). + +This extension deploys 2 BigQuery remote functions, this extension: + +- Perform de-identifaction on sensitive data passed as JSON from BigQuery. +- Re-identify sensitive data that were de-identified with reversable techniques. + +You specify the desired DLP technique. All techniques are powered by the Google [Data Loss Prevention API](https://cloud.google.com/dlp/docs/transformations-reference). The options offered are: + +- Replace with Masking. +- Redact a value (remove it from the data). + +#### Additional setup + +Before installing this extension, make sure that you've set up a BigQuery [dataset](https://cloud.google.com/bigquery/docs/datasets) and [table](https://cloud.google.com/bigquery/docs/tables). + +#### Billing + +This extension uses other Firebase or Google Cloud Platform services which may have associated charges: + +- Cloud Data Loss Prevention API +- BigQuery +- Cloud Functions + +When you use Firebase Extensions, you're only charged for the underlying resources that you use. A paid-tier billing plan is only required if the extension uses a service that requires a paid-tier plan, for example calling to a Google Cloud Platform API or making outbound network requests to non-Google services. All Firebase services offer a free tier of usage. [Learn more about Firebase billing.](https://firebase.google.com/pricing) + **Configuration Parameters:** - DLP Transformation Method: The method used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. @@ -20,6 +66,8 @@ - **deidentifyData:** TODO +- **reidentifyData:** TODO + **APIs Used**: - bigquery.googleapis.com (Reason: Powers all BigQuery tasks performed by the extension.) @@ -40,23 +88,3 @@ This extension will operate with the following project IAM roles: - dlp.user (Reason: Allows the extension to use DLP services.) ---- - -## 🧩 Install this experimental extension - -> ⚠️ **Experimental**: This extension is available for testing as an _experimental_ release. It has not been as thoroughly tested as the officially released extensions, and future updates might introduce breaking changes. If you use this extension, please [report bugs and make feature requests](https://github.com/firebase/experimental-extensions/issues/new/choose) in our GitHub repository. - -### Console - -[![Install this extension in your Firebase project](../install-extension.png?raw=true "Install this extension in your Firebase project")](https://console.firebase.google.com/project/_/extensions/install?ref=firebase/bigquery-dlp-functions) - -### Firebase CLI - -```bash -firebase ext:install firebase/bigquery-dlp-functions --project= -``` - -> Learn more about installing extensions in the Firebase Extensions documentation: [console](https://firebase.google.com/docs/extensions/install-extensions?platform=console), [CLI](https://firebase.google.com/docs/extensions/install-extensions?platform=cli) - ---- - diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index e0908228..1978a0ad 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -97,7 +97,7 @@ exports.createBigQueryConnection = functions.tasks } catch (error) { if (error["code"] === 6) { - functions.logger.info(`Connection ${instanceId} already exists, will continue creating functions`); + functions.logger.warn(`Connection ${instanceId} already exists, will continue creating functions`); } else { functions.logger.error(error); @@ -134,8 +134,14 @@ exports.createBigQueryConnection = functions.tasks await runtime.setProcessingState("PROCESSING_COMPLETE", "Connections created successfully."); } catch (error) { - functions.logger.error(error); - await runtime.setProcessingState("PROCESSING_FAILED", "Connections were not created."); + if (error["code"] === 6) { + functions.logger.warn(`Functions already exists.`); + } + else { + functions.logger.error(error); + await runtime.setProcessingState("PROCESSING_FAILED", "Connections were not created, check logs for more details."); + return; + } } }); //# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index fb91bc7f..90c8d07c 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB,KAAK,QAAQ;oBACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;oBAC5C,MAAM;gBAER;oBACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;aAC7C;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB;oBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/D,OAAO;aACV;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC9E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,UAAU,mDAAmD,CAC5E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,+BAA+B,CAChC,CAAC;KACH;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB,KAAK,QAAQ;oBACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;oBAC5C,MAAM;gBAER;oBACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;aAC7C;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB;oBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/D,OAAO;aACV;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC9E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,UAAU,mDAAmD,CAC5E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACpD;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,4DAA4D,CAC7D,CAAC;YAEF,OAAO;SACR;KACF;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 45a6b925..4e861e6a 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -123,7 +123,7 @@ exports.createBigQueryConnection = functions.tasks functions.logger.info("Connection successfully created 🎉", connection); } catch (error: any) { if (error["code"] === 6) { - functions.logger.info( + functions.logger.warn( `Connection ${instanceId} already exists, will continue creating functions` ); } else { @@ -171,12 +171,18 @@ exports.createBigQueryConnection = functions.tasks "PROCESSING_COMPLETE", "Connections created successfully." ); - } catch (error) { - functions.logger.error(error); + } catch (error: any) { + if (error["code"] === 6) { + functions.logger.warn(`Functions already exists.`); + } else { + functions.logger.error(error); - await runtime.setProcessingState( - "PROCESSING_FAILED", - "Connections were not created." - ); + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Connections were not created, check logs for more details." + ); + + return; + } } }); From e2ba67ff62974c53e9d30f44f64430ed8b486477 Mon Sep 17 00:00:00 2001 From: pr_Mais Date: Thu, 29 Dec 2022 14:16:36 +0300 Subject: [PATCH 29/37] feat: record transformation --- bigquery-dlp-function/extension.yaml | 9 +- bigquery-dlp-function/functions/lib/config.js | 1 + .../functions/lib/config.js.map | 2 +- .../functions/lib/deidentify.js | 102 ++++--------- .../functions/lib/deidentify.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 68 +++++---- .../functions/lib/index.js.map | 2 +- .../functions/lib/transofmrations.js | 95 +++++++++++- .../functions/lib/transofmrations.js.map | 2 +- bigquery-dlp-function/functions/src/config.ts | 1 + .../functions/src/deidentify.ts | 117 +++++---------- bigquery-dlp-function/functions/src/index.ts | 98 ++++++------ .../functions/src/transofmrations.ts | 141 +++++++++++++++--- 13 files changed, 372 insertions(+), 268 deletions(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index 66ce695e..e01185b0 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -70,8 +70,8 @@ params: - label: Info Type Transformations value: INFO_TYPE # TODO support record type transformations - # - label: Record Type Transformations - # value: RECORD + - label: Record Type Transformations + value: RECORD default: INFO_TYPE - param: TRANSFORMATION_TECHNIQUE label: DLP Transformation Technique @@ -84,6 +84,11 @@ params: - label: Redact a value (remove it from the data) value: redact default: masking + - param: FIELDS_TO_TRANSFORM + label: List of fields to transform using record transformation (comma separated) + description: >- + The list of fields to transform using record transformation. This is only used when the transformation method is set to `RECORD`. + type: string - param: DATASET_ID label: BigQuery Dataset ID description: >- diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js index 4d019628..1a8a9315 100644 --- a/bigquery-dlp-function/functions/lib/config.js +++ b/bigquery-dlp-function/functions/lib/config.js @@ -22,5 +22,6 @@ exports.default = { location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, technique: process.env.TRANSFORMATION_TECHNIQUE, + fields: process.env.FIELDS_TO_TRANSFORM, }; //# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map index 018f76e4..4b7871cf 100644 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -1 +1 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;IACzC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;CAChD,CAAC"} \ No newline at end of file +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;IACzC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;IAC/C,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB;CACxC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js b/bigquery-dlp-function/functions/lib/deidentify.js index 94bfa7d0..9c5a52a8 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js +++ b/bigquery-dlp-function/functions/lib/deidentify.js @@ -1,40 +1,8 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.deidentifyWithInfoTypeTransformations = void 0; +exports.deidentifyWithRecordTransformations = exports.deidentifyWithInfoTypeTransformations = void 0; const functions = require("firebase-functions"); -// The maximum number of days to shift a date backward -const lowerBoundDays = 1; -// The maximum number of days to shift a date forward -const upperBoundDays = 1; -function rowsToTable(rows) { - // Helper function for converting rows to Protobuf types - const rowToProto = (row) => { - const values = row.split(","); - const convertedValues = values.map((value) => { - if (Date.parse(value)) { - const date = new Date(value); - return { - dateValue: { - year: date.getFullYear(), - month: date.getMonth() + 1, - day: date.getDate(), - }, - }; - } - else { - // Convert all non-date values to strings - return { stringValue: value.toString() }; - } - }); - return { values: convertedValues }; - }; - // Construct the table object - const table = { - headers: [], - rows: rows.map((row) => rowToProto(row[0])), - }; - return table; -} +const transofmrations_1 = require("./transofmrations"); /** * Deidentify sensitive data in a string with [the Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) * using `infoTypeTransformations` method. @@ -43,14 +11,13 @@ function rowsToTable(rows) { * * @param {string} text The text to deidentify. * - * @returns {string} The deidentified text. + * @returns {Promise>} The deidentified record. */ async function deidentifyWithInfoTypeTransformations(rows, client, transformation) { var _a; const deidentifiedItems = []; for (const row of rows) { const data = row[0]; - functions.logger.debug(data); for (const key in data) { if (data.hasOwnProperty(key)) { const element = data[key]; @@ -59,7 +26,6 @@ async function deidentifyWithInfoTypeTransformations(rows, client, transformatio data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; } } - functions.logger.debug(data); deidentifiedItems.push(data); } return deidentifiedItems; @@ -73,45 +39,27 @@ exports.deidentifyWithInfoTypeTransformations = deidentifyWithInfoTypeTransforma * * @param {string} text The text to deidentify. * - * @returns {string} The deidentified text. + * @returns {Promise} The deidentified text. */ -// export async function deidentifyWithRecordTransformations( -// rows: any, -// client: DlpServiceClient, -// mask?: string, -// numberToMask?: number -// ) { -// const parent = `projects/${config.projectId}/locations/${config.location}`; -// // Construct DateShiftConfig -// const dateShiftConfig = { -// lowerBoundDays: lowerBoundDays, -// upperBoundDays: upperBoundDays, -// }; -// // Construct de-identification request -// const request: DeidentifyRequest = { -// parent: parent, -// deidentifyConfig: { -// recordTransformations: { -// fieldTransformations: [ -// { -// primitiveTransformation: { -// dateShiftConfig: dateShiftConfig, -// characterMaskConfig: { -// maskingCharacter: mask ?? "x", -// numberToMask: numberToMask ?? 5, -// }, -// }, -// }, -// ], -// }, -// }, -// item: { -// table: rowsToTable(rows), -// }, -// }; -// // Run deidentification request -// const [response] = await client.deidentifyContent(request); -// const tableRows = response.item?.table?.rows; -// return tableRows; -// } +async function deidentifyWithRecordTransformations(rows, client, transformation) { + var _a, _b; + let table; + try { + // Convert raw rows to Table type + table = (0, transofmrations_1.rowsToTable)(rows); + } + catch (error) { + functions.logger.debug(`Error converting rows to Table type.`); + throw error; + } + // Construct de-identification request + const request = Object.assign(Object.assign({}, transformation.deidentifyConfig), { item: { + table: table, + } }); + // Run deidentification request + const [response] = await client.deidentifyContent(request); + functions.logger.debug((0, transofmrations_1.tableToReplies)((_a = response.item) === null || _a === void 0 ? void 0 : _a.table)); + return (0, transofmrations_1.tableToReplies)((_b = response.item) === null || _b === void 0 ? void 0 : _b.table); +} +exports.deidentifyWithRecordTransformations = deidentifyWithRecordTransformations; //# sourceMappingURL=deidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js.map b/bigquery-dlp-function/functions/lib/deidentify.js.map index b7ca7ada..55f4380a 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js.map +++ b/bigquery-dlp-function/functions/lib/deidentify.js.map @@ -1 +1 @@ -{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAKhD,sDAAsD;AACtD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,qDAAqD;AACrD,MAAM,cAAc,GAAG,CAAC,CAAC;AAEzB,SAAS,WAAW,CAAC,IAAQ;IAC3B,wDAAwD;IACxD,MAAM,UAAU,GAAG,CAAC,GAAQ,EAAE,EAAE;QAC9B,MAAM,MAAM,GAAG,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QAC9B,MAAM,eAAe,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAU,EAAE,EAAE;YAChD,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,EAAE;gBACrB,MAAM,IAAI,GAAG,IAAI,IAAI,CAAC,KAAK,CAAC,CAAC;gBAC7B,OAAO;oBACL,SAAS,EAAE;wBACT,IAAI,EAAE,IAAI,CAAC,WAAW,EAAE;wBACxB,KAAK,EAAE,IAAI,CAAC,QAAQ,EAAE,GAAG,CAAC;wBAC1B,GAAG,EAAE,IAAI,CAAC,OAAO,EAAE;qBACpB;iBACF,CAAC;aACH;iBAAM;gBACL,yCAAyC;gBACzC,OAAO,EAAE,WAAW,EAAE,KAAK,CAAC,QAAQ,EAAE,EAAE,CAAC;aAC1C;QACH,CAAC,CAAC,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IACrC,CAAC,CAAC;IAEF,6BAA6B;IAC7B,MAAM,KAAK,GAAG;QACZ,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;KAC5C,CAAC;IAEF,OAAO,KAAK,CAAC;AACf,CAAC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACzB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA7BD,sFA6BC;AAED;;;;;;;;;GASG;AACH,6DAA6D;AAC7D,eAAe;AACf,8BAA8B;AAC9B,mBAAmB;AACnB,0BAA0B;AAC1B,MAAM;AACN,gFAAgF;AAEhF,iCAAiC;AACjC,8BAA8B;AAC9B,sCAAsC;AACtC,sCAAsC;AACtC,OAAO;AAEP,2CAA2C;AAC3C,yCAAyC;AACzC,sBAAsB;AACtB,0BAA0B;AAC1B,iCAAiC;AACjC,kCAAkC;AAClC,cAAc;AACd,yCAAyC;AACzC,kDAAkD;AAClD,uCAAuC;AACvC,iDAAiD;AACjD,mDAAmD;AACnD,mBAAmB;AACnB,iBAAiB;AACjB,eAAe;AACf,aAAa;AACb,WAAW;AACX,SAAS;AACT,cAAc;AACd,kCAAkC;AAClC,SAAS;AACT,OAAO;AAEP,oCAAoC;AACpC,gEAAgE;AAChE,kDAAkD;AAElD,sBAAsB;AACtB,IAAI"} \ No newline at end of file +{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,uDAK2B;AAE3B;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACzB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA3BD,sFA2BC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mCAAmC,CACvD,IAAS,EACT,MAAwB,EACxB,cAAyD;;IAEzD,IAAI,KAAK,CAAC;IAEV,IAAI;QACF,iCAAiC;QACjC,KAAK,GAAG,IAAA,6BAAW,EAAC,IAAI,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;KACb;IAED,sCAAsC;IACtC,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE;YACJ,KAAK,EAAE,KAAK;SACb,GACF,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC,CAAC;IAE7D,OAAO,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AA7BD,kFA6BC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 1978a0ad..813a1bc4 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -17,61 +17,63 @@ const bigqueryConnectionClient = new bigquery_connection_1.ConnectionServiceClie exports.deidentifyData = functions.https.onRequest(async (request, response) => { const { calls, userDefinedContext } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); + var transformation; + switch (config_1.default.technique) { + case "redact": + transformation = new transofmrations_1.RedactTransformation(); + break; + default: + transformation = new transofmrations_1.MaskTransformation(); + } try { - if (userDefinedContext.method === "INFO_TYPE") { - var transformation; - switch (config_1.default.technique) { - case "redact": - transformation = new transofmrations_1.RedactTransformation(); - break; - default: - transformation = new transofmrations_1.MaskTransformation(); - } - response.send({ - replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, transformation), - }); - } - // else if (userDefinedContext.method === "RECORD") { - // response.send({ - // replies: await deidentifyWithRecordTransformations(calls, dlp), - // }); - // } - else { - response.status(400).send("Invalid method"); + switch (userDefinedContext.method) { + case "INFO_TYPE": + response.send({ + replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, transformation), + }); + break; + case "RECORD": + response.send({ + replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp, transformation), + }); + break; + default: + response.status(400).send({ errorMessage: "Invalid method" }); + break; } } catch (error) { functions.logger.error(error); - response.status(500).send(`errorMessage: ${error}`); + response.status(400).send({ errorMessage: error }); } }); exports.reidentifyData = functions.https.onRequest(async (request, response) => { const { calls, userDefinedContext } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); + var transformation; + switch (config_1.default.technique) { + default: + response.status(400).send("Invalid or irreversable technique"); + return; + } try { if (userDefinedContext.method === "INFO_TYPE") { - var transformation; - switch (config_1.default.technique) { - default: - response.status(400).send("Invalid or irreversable technique"); - return; - } response.send({ replies: await (0, reidentify_1.reidentifyWithInfoTypeTransformations)(calls, dlp, transformation), }); } - // else if (userDefinedContext.method === "RECORD") { - // response.send({ - // replies: await deidentifyWithRecordTransformations(calls, dlp), - // }); - // } + else if (userDefinedContext.method === "RECORD") { + response.send({ + replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp, transformation), + }); + } else { response.status(400).send("Invalid method"); } } catch (error) { functions.logger.error(error); - response.status(500).send(`errorMessage: ${error}`); + response.status(400).send({ errorMessage: error }); } }); exports.createBigQueryConnection = functions.tasks diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 90c8d07c..70867fb4 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAAqE;AACrE,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB,KAAK,QAAQ;oBACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;oBAC5C,MAAM;gBAER;oBACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;aAC7C;YACD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,IAAI,cAAc,CAAC;YAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;gBACxB;oBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;oBAC/D,OAAO;aACV;YAED,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;QACD,qDAAqD;QACrD,oBAAoB;QACpB,sEAAsE;QACtE,QAAQ;QACR,IAAI;aACC;YACH,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,iBAAiB,KAAK,EAAE,CAAC,CAAC;KACrD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC9E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,UAAU,mDAAmD,CAC5E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACpD;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,4DAA4D,CAC7D,CAAC;YAEF,OAAO;SACR;KACF;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AACtB,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACxB,KAAK,QAAQ;YACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;YAC5C,MAAM;QAER;YACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;KAC7C;IAED,IAAI;QACF,QAAQ,kBAAkB,CAAC,MAAM,EAAE;YACjC,KAAK,WAAW;gBACd,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;iBACF,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,QAAQ;gBACX,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAChD,KAAK,EACL,GAAG,EACH,cAAc,CACf;iBACF,CAAC,CAAC;gBACH,MAAM;YACR;gBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC9D,MAAM;SACT;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACpD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACxB;YACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC/D,OAAO;KACV;IAED,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAChD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACpD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC9E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,UAAU,mDAAmD,CAC5E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACpD;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,4DAA4D,CAC7D,CAAC;YAEF,OAAO;SACR;KACF;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js index a26393ce..65466425 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js +++ b/bigquery-dlp-function/functions/lib/transofmrations.js @@ -1,7 +1,8 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.RedactTransformation = exports.MaskTransformation = void 0; +exports.tableToReplies = exports.getFieldIds = exports.rowsToTable = exports.RedactTransformation = exports.MaskTransformation = void 0; const config_1 = require("./config"); +const dlp_1 = require("@google-cloud/dlp"); class Transformation { constructor() { this.deidentifyConfig = {}; @@ -18,7 +19,7 @@ class MaskTransformation extends Transformation { */ constructor(mask, numberToMask) { super(); - const maskingConfig = { + const maskingConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { infoTypeTransformations: { transformations: [ { @@ -31,7 +32,21 @@ class MaskTransformation extends Transformation { }, ], }, - }; + })), (config_1.default.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", + numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, + }, + }, + }, + ], + }, + })); this.deidentifyConfig = { parent: this.parent, deidentifyConfig: maskingConfig, @@ -45,7 +60,7 @@ class RedactTransformation extends Transformation { */ constructor() { super(); - const maskingConfig = { + const redactConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { infoTypeTransformations: { transformations: [ { @@ -55,12 +70,80 @@ class RedactTransformation extends Transformation { }, ], }, - }; + })), (config_1.default.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + })); this.deidentifyConfig = { parent: this.parent, - deidentifyConfig: maskingConfig, + deidentifyConfig: redactConfig, }; } } exports.RedactTransformation = RedactTransformation; +function rowsToTable(rows) { + let table = { + headers: [], + rows: [], + }; + for (const row of rows) { + const data = row[0]; + const keys = Object.keys(data); + const values = Object.values(data); + if (table.headers.length === 0) { + // Add headers only once + table.headers = keys.map((key) => { + const field = dlp_1.protos.google.privacy.dlp.v2.FieldId.create({ + name: key, + }); + return field; + }); + } + const tableRow = dlp_1.protos.google.privacy.dlp.v2.Table.Row.create({ + values: values.map((v) => { + const field = dlp_1.protos.google.privacy.dlp.v2.Value.create({ + stringValue: v, + }); + return field; + }), + }); + table.rows.push(tableRow); + } + return table; +} +exports.rowsToTable = rowsToTable; +function getFieldIds() { + var _a; + const fields = (_a = config_1.default.fields) === null || _a === void 0 ? void 0 : _a.split(","); + const fieldIds = fields === null || fields === void 0 ? void 0 : fields.map((field) => { + return { name: field }; + }); + return fieldIds; +} +exports.getFieldIds = getFieldIds; +function tableToReplies(table) { + var _a; + const replies = []; + const rows = (_a = table === null || table === void 0 ? void 0 : table.rows) === null || _a === void 0 ? void 0 : _a.map((row) => { var _a; return (_a = row.values) === null || _a === void 0 ? void 0 : _a.map((value) => value.stringValue); }); + if (!rows || !table || !table.headers) + return []; + for (const row of rows) { + const reply = {}; + for (let i = 0; i < table.headers.length; i++) { + const header = table.headers[i].name; + reply[header] = row[i]; + } + replies.push(reply); + } + return replies; +} +exports.tableToReplies = tableToReplies; //# sourceMappingURL=transofmrations.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js.map b/bigquery-dlp-function/functions/lib/transofmrations.js.map index 9a4d02ef..b674933d 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js.map +++ b/bigquery-dlp-function/functions/lib/transofmrations.js.map @@ -1 +1 @@ -{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAM9B,MAAM,cAAc;IAKlB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGvC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC5E,CAAC;CACF;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC9C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AA7BD,gDA6BC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACtD;;OAEG;IACH;QACE,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,GAAG;YACpB,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AAvBD,oDAuBC"} \ No newline at end of file +{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAC9B,2CAA2C;AAQ3C,MAAM,cAAc;IAKlB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGvC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC5E,CAAC;CACF;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC9C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,mCACd,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YAClC,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAC/B,qBAAqB,EAAE;gBACrB,oBAAoB,EAAE;oBACpB;wBACE,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AA9CD,gDA8CC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACtD;;OAEG;IACH;QACE,KAAK,EAAE,CAAC;QACR,MAAM,YAAY,mCACb,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YAClC,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAC/B,qBAAqB,EAAE;gBACrB,oBAAoB,EAAE;oBACpB;wBACE,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,YAAY;SAC/B,CAAC;IACJ,CAAC;CACF;AArCD,oDAqCC;AAED,SAAgB,WAAW,CAAC,IAAQ;IAClC,IAAI,KAAK,GAAG;QACV,OAAO,EAAE,EAAe;QACxB,IAAI,EAAE,EAAW;KAClB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,wBAAwB;YACxB,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;oBACxD,IAAI,EAAE,GAAG;iBACV,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;SACJ;QAED,MAAM,QAAQ,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAC7D,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACvB,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;oBACtD,WAAW,EAAE,CAAC;iBACf,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACf,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC3B;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAnCD,kCAmCC;AAED,SAAgB,WAAW;;IACzB,MAAM,MAAM,GAAG,MAAA,gBAAM,CAAC,MAAM,0CAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACrC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAND,kCAMC;AAED,SAAgB,cAAc,CAAC,KAAY;;IACzC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,0CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WACpC,OAAA,MAAA,GAAG,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA,EAAA,CAC9C,CAAC;IAEF,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,KAAK,GAAG,EAAyB,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC;YAC/C,KAAK,CAAC,MAAM,CAAC,GAAG,GAAI,CAAC,CAAC,CAAC,CAAC;SACzB;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACrB;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AApBD,wCAoBC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 6dad6dd0..54399b00 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -21,4 +21,5 @@ export default { location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, technique: process.env.TRANSFORMATION_TECHNIQUE, + fields: process.env.FIELDS_TO_TRANSFORM, }; diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index 95acae5f..acdb9897 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -1,44 +1,12 @@ import * as functions from "firebase-functions"; import { DlpServiceClient } from "@google-cloud/dlp"; -import { MaskTransformation, RedactTransformation } from "./transofmrations"; - -// The maximum number of days to shift a date backward -const lowerBoundDays = 1; - -// The maximum number of days to shift a date forward -const upperBoundDays = 1; - -function rowsToTable(rows: []) { - // Helper function for converting rows to Protobuf types - const rowToProto = (row: any) => { - const values = row.split(","); - const convertedValues = values.map((value: any) => { - if (Date.parse(value)) { - const date = new Date(value); - return { - dateValue: { - year: date.getFullYear(), - month: date.getMonth() + 1, - day: date.getDate(), - }, - }; - } else { - // Convert all non-date values to strings - return { stringValue: value.toString() }; - } - }); - return { values: convertedValues }; - }; - - // Construct the table object - const table = { - headers: [], - rows: rows.map((row) => rowToProto(row[0])), - }; - - return table; -} +import { + MaskTransformation, + RedactTransformation, + rowsToTable, + tableToReplies, +} from "./transofmrations"; /** * Deidentify sensitive data in a string with [the Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) @@ -48,18 +16,17 @@ function rowsToTable(rows: []) { * * @param {string} text The text to deidentify. * - * @returns {string} The deidentified text. + * @returns {Promise>} The deidentified record. */ export async function deidentifyWithInfoTypeTransformations( rows: [], client: DlpServiceClient, transformation: MaskTransformation | RedactTransformation -) { +): Promise> { const deidentifiedItems = []; for (const row of rows) { const data = row[0] as Record; - functions.logger.debug(data); for (const key in data) { if (data.hasOwnProperty(key)) { @@ -74,7 +41,6 @@ export async function deidentifyWithInfoTypeTransformations( } } - functions.logger.debug(data); deidentifiedItems.push(data); } @@ -89,48 +55,35 @@ export async function deidentifyWithInfoTypeTransformations( * * @param {string} text The text to deidentify. * - * @returns {string} The deidentified text. + * @returns {Promise} The deidentified text. */ -// export async function deidentifyWithRecordTransformations( -// rows: any, -// client: DlpServiceClient, -// mask?: string, -// numberToMask?: number -// ) { -// const parent = `projects/${config.projectId}/locations/${config.location}`; +export async function deidentifyWithRecordTransformations( + rows: any, + client: DlpServiceClient, + transformation: MaskTransformation | RedactTransformation +) { + let table; + + try { + // Convert raw rows to Table type + table = rowsToTable(rows); + } catch (error) { + functions.logger.debug(`Error converting rows to Table type.`); + throw error; + } -// // Construct DateShiftConfig -// const dateShiftConfig = { -// lowerBoundDays: lowerBoundDays, -// upperBoundDays: upperBoundDays, -// }; + // Construct de-identification request + const request = { + ...transformation.deidentifyConfig, + item: { + table: table, + }, + }; -// // Construct de-identification request -// const request: DeidentifyRequest = { -// parent: parent, -// deidentifyConfig: { -// recordTransformations: { -// fieldTransformations: [ -// { -// primitiveTransformation: { -// dateShiftConfig: dateShiftConfig, -// characterMaskConfig: { -// maskingCharacter: mask ?? "x", -// numberToMask: numberToMask ?? 5, -// }, -// }, -// }, -// ], -// }, -// }, -// item: { -// table: rowsToTable(rows), -// }, -// }; + // Run deidentification request + const [response] = await client.deidentifyContent(request); -// // Run deidentification request -// const [response] = await client.deidentifyContent(request); -// const tableRows = response.item?.table?.rows; + functions.logger.debug(tableToReplies(response.item?.table)); -// return tableRows; -// } + return tableToReplies(response.item?.table); +} diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 4e861e6a..452a9151 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -6,7 +6,10 @@ import { BigQuery } from "@google-cloud/bigquery"; import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; -import { deidentifyWithInfoTypeTransformations } from "./deidentify"; +import { + deidentifyWithInfoTypeTransformations, + deidentifyWithRecordTransformations, +} from "./deidentify"; import { MaskTransformation, RedactTransformation } from "./transofmrations"; import { reidentifyWithInfoTypeTransformations } from "./reidentify"; @@ -21,39 +24,45 @@ exports.deidentifyData = functions.https.onRequest( const { calls, userDefinedContext } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); + var transformation; - try { - if (userDefinedContext.method === "INFO_TYPE") { - var transformation; + switch (config.technique) { + case "redact": + transformation = new RedactTransformation(); + break; - switch (config.technique) { - case "redact": - transformation = new RedactTransformation(); - break; + default: + transformation = new MaskTransformation(); + } - default: - transformation = new MaskTransformation(); - } - response.send({ - replies: await deidentifyWithInfoTypeTransformations( - calls, - dlp, - transformation - ), - }); - } - // else if (userDefinedContext.method === "RECORD") { - // response.send({ - // replies: await deidentifyWithRecordTransformations(calls, dlp), - // }); - // } - else { - response.status(400).send("Invalid method"); + try { + switch (userDefinedContext.method) { + case "INFO_TYPE": + response.send({ + replies: await deidentifyWithInfoTypeTransformations( + calls, + dlp, + transformation + ), + }); + break; + case "RECORD": + response.send({ + replies: await deidentifyWithRecordTransformations( + calls, + dlp, + transformation + ), + }); + break; + default: + response.status(400).send({ errorMessage: "Invalid method" }); + break; } } catch (error) { functions.logger.error(error); - response.status(500).send(`errorMessage: ${error}`); + response.status(400).send({ errorMessage: error }); } } ); @@ -64,16 +73,16 @@ exports.reidentifyData = functions.https.onRequest( functions.logger.debug("Incoming request from BigQuery", calls); - try { - if (userDefinedContext.method === "INFO_TYPE") { - var transformation; + var transformation; - switch (config.technique) { - default: - response.status(400).send("Invalid or irreversable technique"); - return; - } + switch (config.technique) { + default: + response.status(400).send("Invalid or irreversable technique"); + return; + } + try { + if (userDefinedContext.method === "INFO_TYPE") { response.send({ replies: await reidentifyWithInfoTypeTransformations( calls, @@ -81,19 +90,20 @@ exports.reidentifyData = functions.https.onRequest( transformation ), }); - } - // else if (userDefinedContext.method === "RECORD") { - // response.send({ - // replies: await deidentifyWithRecordTransformations(calls, dlp), - // }); - // } - else { + } else if (userDefinedContext.method === "RECORD") { + response.send({ + replies: await deidentifyWithRecordTransformations( + calls, + dlp, + transformation + ), + }); + } else { response.status(400).send("Invalid method"); } } catch (error) { functions.logger.error(error); - - response.status(500).send(`errorMessage: ${error}`); + response.status(400).send({ errorMessage: error }); } } ); diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts index 87ebb610..a48f4dfe 100644 --- a/bigquery-dlp-function/functions/src/transofmrations.ts +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -3,6 +3,9 @@ import { protos } from "@google-cloud/dlp"; type ReidentifyRequest = protos.google.privacy.dlp.v2.IReidentifyContentRequest; type DeidentifyRequest = protos.google.privacy.dlp.v2.IDeidentifyContentRequest; +type FieldId = protos.google.privacy.dlp.v2.FieldId; +type Row = protos.google.privacy.dlp.v2.Table.Row; +type Table = protos.google.privacy.dlp.v2.ITable | undefined | null; class Transformation { parent: string; @@ -24,18 +27,35 @@ export class MaskTransformation extends Transformation { constructor(mask?: string, numberToMask?: number) { super(); const maskingConfig = { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, }, }, - }, - ], - }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }), }; this.deidentifyConfig = { @@ -51,21 +71,102 @@ export class RedactTransformation extends Transformation { */ constructor() { super(); - const maskingConfig = { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - redactConfig: {}, + const redactConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + redactConfig: {}, + }, }, - }, - ], - }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }), }; this.deidentifyConfig = { parent: this.parent, - deidentifyConfig: maskingConfig, + deidentifyConfig: redactConfig, }; } } + +export function rowsToTable(rows: []) { + let table = { + headers: [] as FieldId[], + rows: [] as Row[], + }; + + for (const row of rows) { + const data = row[0] as Record; + + const keys = Object.keys(data); + const values = Object.values(data); + + if (table.headers.length === 0) { + // Add headers only once + table.headers = keys.map((key) => { + const field = protos.google.privacy.dlp.v2.FieldId.create({ + name: key, + }); + return field; + }); + } + + const tableRow = protos.google.privacy.dlp.v2.Table.Row.create({ + values: values.map((v) => { + const field = protos.google.privacy.dlp.v2.Value.create({ + stringValue: v, + }); + return field; + }), + }); + + table.rows.push(tableRow); + } + + return table; +} + +export function getFieldIds() { + const fields = config.fields?.split(","); + const fieldIds = fields?.map((field) => { + return { name: field }; + }); + return fieldIds; +} + +export function tableToReplies(table: Table) { + const replies = []; + const rows = table?.rows?.map((row) => + row.values?.map((value) => value.stringValue) + ); + + if (!rows || !table || !table.headers) return []; + + for (const row of rows) { + const reply = {} as Record; + + for (let i = 0; i < table.headers.length; i++) { + const header = table.headers[i].name as string; + reply[header] = row![i]; + } + + replies.push(reply); + } + + return replies; +} From 2e3ff75affe725faf94dfcc2c7e24bf55c13405f Mon Sep 17 00:00:00 2001 From: dackers86 Date: Thu, 29 Dec 2022 16:54:31 +0000 Subject: [PATCH 30/37] chore(bigquery-dlp-function): updated extension version and chnagelog --- bigquery-dlp-function/CHANGELOG.md | 4 ++++ bigquery-dlp-function/extension.yaml | 2 +- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/bigquery-dlp-function/CHANGELOG.md b/bigquery-dlp-function/CHANGELOG.md index 6f0a5d30..af3bf27c 100644 --- a/bigquery-dlp-function/CHANGELOG.md +++ b/bigquery-dlp-function/CHANGELOG.md @@ -1,3 +1,7 @@ +## Version 0.0.3 + +feat: added record transformations + ## Version 0.0.2 fix: remove default value in DATASET_ID param diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index e01185b0..d942ba13 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -2,7 +2,7 @@ # https://firebase.google.com/docs/extensions/alpha/ref-extension-yaml name: bigquery-dlp-functions # Identifier for your extension -version: 0.0.2 # Follow semver versioning +version: 0.0.3 # Follow semver versioning specVersion: v1beta # Version of the Firebase Extensions specification author: From 97c12965a1fad720ca7659b53623ac82073ef6e8 Mon Sep 17 00:00:00 2001 From: Mais Date: Tue, 28 Feb 2023 17:41:47 +0300 Subject: [PATCH 31/37] feat: added replace transformations --- bigquery-dlp-function/extension.yaml | 9 +- bigquery-dlp-function/functions/lib/index.js | 6 + .../functions/lib/index.js.map | 2 +- .../functions/lib/transofmrations.js | 83 +++- .../functions/lib/transofmrations.js.map | 2 +- bigquery-dlp-function/functions/src/index.ts | 324 +++++++-------- .../functions/src/transofmrations.ts | 372 +++++++++++------- 7 files changed, 495 insertions(+), 303 deletions(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index d942ba13..e241cda1 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -69,10 +69,10 @@ params: options: - label: Info Type Transformations value: INFO_TYPE - # TODO support record type transformations - label: Record Type Transformations value: RECORD default: INFO_TYPE + - param: TRANSFORMATION_TECHNIQUE label: DLP Transformation Technique description: >- @@ -83,12 +83,18 @@ params: value: masking - label: Redact a value (remove it from the data) value: redact + - label: Replace with a fixed value + value: fixed + - label: Replace with InfoType value + value: replaceWithInfoType default: masking + - param: FIELDS_TO_TRANSFORM label: List of fields to transform using record transformation (comma separated) description: >- The list of fields to transform using record transformation. This is only used when the transformation method is set to `RECORD`. type: string + - param: DATASET_ID label: BigQuery Dataset ID description: >- @@ -96,6 +102,7 @@ params: type: string required: true immutable: true + - param: LOCATION label: Cloud Functions location description: >- diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 813a1bc4..6c0a8be1 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -22,6 +22,12 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => case "redact": transformation = new transofmrations_1.RedactTransformation(); break; + case "fixed": + transformation = new transofmrations_1.ReplaceTransformation(); + break; + case "replaceWithInfoType": + transformation = new transofmrations_1.ReplaceWithInfoTypeTransformation(); + break; default: transformation = new transofmrations_1.MaskTransformation(); } diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index 70867fb4..acc6a65f 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AACtB,uDAA6E;AAC7E,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACxB,KAAK,QAAQ;YACX,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;YAC5C,MAAM;QAER;YACE,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;KAC7C;IAED,IAAI;QACF,QAAQ,kBAAkB,CAAC,MAAM,EAAE;YACjC,KAAK,WAAW;gBACd,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;iBACF,CAAC,CAAC;gBACH,MAAM;YACR,KAAK,QAAQ;gBACX,QAAQ,CAAC,IAAI,CAAC;oBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAChD,KAAK,EACL,GAAG,EACH,cAAc,CACf;iBACF,CAAC,CAAC;gBACH,MAAM;YACR;gBACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC9D,MAAM;SACT;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACpD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CAChD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC1B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACxB;YACE,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC/D,OAAO;KACV;IAED,IAAI;QACF,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC7C,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,kDAAqC,EAClD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YACjD,QAAQ,CAAC,IAAI,CAAC;gBACZ,OAAO,EAAE,MAAM,IAAA,gDAAmC,EAChD,KAAK,EACL,GAAG,EACH,cAAc,CACf;aACF,CAAC,CAAC;SACJ;aAAM;YACL,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC7C;KACF;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACpD;AACH,CAAC,CACF,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAC/C,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACrB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACF,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC3D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACV,aAAa,EAAE;oBACb,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC9E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC9B;SACF,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACzE;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CACnB,cAAc,UAAU,mDAAmD,CAC5E,CAAC;SACH;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,yDAAyD,CAC1D,CAAC;YAEF,OAAO;SACR;KACF;IAED,IAAI;QACF,MAAM,KAAK,GAAG;;8BAEU,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAEL,MAAM,OAAO,GAAG;YACd,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SAC1B,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,qBAAqB,EACrB,mCAAmC,CACpC,CAAC;KACH;IAAC,OAAO,KAAU,EAAE;QACnB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACvB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACpD;aAAM;YACL,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC9B,mBAAmB,EACnB,4DAA4D,CAC7D,CAAC;YAEF,OAAO;SACR;KACF;AACH,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AACtB,uDAK2B;AAC3B,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB,KAAK,QAAQ;YACZ,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;YAC5C,MAAM;QACP,KAAK,OAAO;YACX,cAAc,GAAG,IAAI,uCAAqB,EAAE,CAAC;YAC7C,MAAM;QACP,KAAK,qBAAqB;YACzB,cAAc,GAAG,IAAI,mDAAiC,EAAE,CAAC;YACzD,MAAM;QACP;YACC,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;KAC3C;IAED,IAAI;QACH,QAAQ,kBAAkB,CAAC,MAAM,EAAE;YAClC,KAAK,WAAW;gBACf,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP,KAAK,QAAQ;gBACZ,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP;gBACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC9D,MAAM;SACP;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB;YACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC/D,OAAO;KACR;IAED,IAAI;QACH,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC9C,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YAClD,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM;YACN,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC5C;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAChD,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACtB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACH,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC5D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACX,aAAa,EAAE;oBACd,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC7E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC7B;SACD,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACxE;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CACpB,cAAc,UAAU,mDAAmD,CAC3E,CAAC;SACF;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,yDAAyD,CACzD,CAAC;YAEF,OAAO;SACP;KACD;IAED,IAAI;QACH,MAAM,KAAK,GAAG;;8BAEa,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAER,MAAM,OAAO,GAAG;YACf,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SACzB,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,qBAAqB,EACrB,mCAAmC,CACnC,CAAC;KACF;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACnD;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,4DAA4D,CAC5D,CAAC;YAEF,OAAO;SACP;KACD;AACF,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js index 65466425..4ec90c82 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js +++ b/bigquery-dlp-function/functions/lib/transofmrations.js @@ -1,6 +1,6 @@ "use strict"; Object.defineProperty(exports, "__esModule", { value: true }); -exports.tableToReplies = exports.getFieldIds = exports.rowsToTable = exports.RedactTransformation = exports.MaskTransformation = void 0; +exports.tableToReplies = exports.rowsToTable = exports.ReplaceWithInfoTypeTransformation = exports.ReplaceTransformation = exports.RedactTransformation = exports.MaskTransformation = void 0; const config_1 = require("./config"); const dlp_1 = require("@google-cloud/dlp"); class Transformation { @@ -89,6 +89,86 @@ class RedactTransformation extends Transformation { } } exports.RedactTransformation = RedactTransformation; +class ReplaceTransformation extends Transformation { + /** + * Replace with a specified value. + */ + constructor() { + super(); + const _replaceConfig = { + newValue: { + // TODO make configurable? + stringValue: "REPLACED", + }, + }; + const replaceConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + replaceConfig: _replaceConfig, + }, + }, + ], + }, + })), (config_1.default.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + replaceConfig: _replaceConfig, + }, + }, + ], + }, + })); + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: replaceConfig, + }; + } +} +exports.ReplaceTransformation = ReplaceTransformation; +class ReplaceWithInfoTypeTransformation extends Transformation { + /** + * Replace with a specified value. + */ + constructor() { + super(); + const _replaceConfig = { + // TODO make configurable? + partToExtract: "MONTH", + }; + const replaceConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + replaceWithInfoTypeConfig: _replaceConfig, + }, + }, + ], + }, + })), (config_1.default.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + replaceWithInfoTypeConfig: _replaceConfig, + }, + }, + ], + }, + })); + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: replaceConfig, + }; + } +} +exports.ReplaceWithInfoTypeTransformation = ReplaceWithInfoTypeTransformation; function rowsToTable(rows) { let table = { headers: [], @@ -128,7 +208,6 @@ function getFieldIds() { }); return fieldIds; } -exports.getFieldIds = getFieldIds; function tableToReplies(table) { var _a; const replies = []; diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js.map b/bigquery-dlp-function/functions/lib/transofmrations.js.map index b674933d..6dc95ece 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js.map +++ b/bigquery-dlp-function/functions/lib/transofmrations.js.map @@ -1 +1 @@ -{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAC9B,2CAA2C;AAQ3C,MAAM,cAAc;IAKlB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGvC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC5E,CAAC;CACF;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACpD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC9C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,mCACd,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YAClC,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAC/B,qBAAqB,EAAE;gBACrB,oBAAoB,EAAE;oBACpB;wBACE,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACvB,mBAAmB,EAAE;gCACnB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAChC;yBACF;qBACF;iBACF;aACF;SACF,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAChC,CAAC;IACJ,CAAC;CACF;AA9CD,gDA8CC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACtD;;OAEG;IACH;QACE,KAAK,EAAE,CAAC;QACR,MAAM,YAAY,mCACb,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YAClC,uBAAuB,EAAE;gBACvB,eAAe,EAAE;oBACf;wBACE,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAC/B,qBAAqB,EAAE;gBACrB,oBAAoB,EAAE;oBACpB;wBACE,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACvB,YAAY,EAAE,EAAE;yBACjB;qBACF;iBACF;aACF;SACF,CAAC,CACH,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACtB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,YAAY;SAC/B,CAAC;IACJ,CAAC;CACF;AArCD,oDAqCC;AAED,SAAgB,WAAW,CAAC,IAAQ;IAClC,IAAI,KAAK,GAAG;QACV,OAAO,EAAE,EAAe;QACxB,IAAI,EAAE,EAAW;KAClB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC9B,wBAAwB;YACxB,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAC/B,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;oBACxD,IAAI,EAAE,GAAG;iBACV,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACf,CAAC,CAAC,CAAC;SACJ;QAED,MAAM,QAAQ,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAC7D,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACvB,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;oBACtD,WAAW,EAAE,CAAC;iBACf,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACf,CAAC,CAAC;SACH,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC3B;IAED,OAAO,KAAK,CAAC;AACf,CAAC;AAnCD,kCAmCC;AAED,SAAgB,WAAW;;IACzB,MAAM,MAAM,GAAG,MAAA,gBAAM,CAAC,MAAM,0CAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACrC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACzB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AAClB,CAAC;AAND,kCAMC;AAED,SAAgB,cAAc,CAAC,KAAY;;IACzC,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,0CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WACpC,OAAA,MAAA,GAAG,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA,EAAA,CAC9C,CAAC;IAEF,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,KAAK,GAAG,EAAyB,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC7C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC;YAC/C,KAAK,CAAC,MAAM,CAAC,GAAG,GAAI,CAAC,CAAC,CAAC,CAAC;SACzB;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACrB;IAED,OAAO,OAAO,CAAC;AACjB,CAAC;AApBD,wCAoBC"} \ No newline at end of file +{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAC9B,2CAA2C;AAQ3C,MAAM,cAAc;IAKnB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGxC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,CAAC;CACD;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACrD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC/C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA9CD,gDA8CC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACvD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,YAAY,mCACd,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,YAAY;SAC9B,CAAC;IACH,CAAC;CACD;AArCD,oDAqCC;AAED,MAAa,qBAAsB,SAAQ,cAAc;IACxD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,QAAQ,EAAE;gBACT,0BAA0B;gBAC1B,WAAW,EAAE,UAAU;aACvB;SACD,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA5CD,sDA4CC;AAED,MAAa,iCAAkC,SAAQ,cAAc;IACpE;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,0BAA0B;YAC1B,aAAa,EAAE,OAAO;SACtB,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA1CD,8EA0CC;AAED,SAAgB,WAAW,CAAC,IAAQ;IACnC,IAAI,KAAK,GAAG;QACX,OAAO,EAAE,EAAe;QACxB,IAAI,EAAE,EAAW;KACjB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/B,wBAAwB;YACxB,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAChC,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;oBACzD,IAAI,EAAE,GAAG;iBACT,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAC9D,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxB,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;oBACvD,WAAW,EAAE,CAAC;iBACd,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC;SACF,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC1B;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAnCD,kCAmCC;AAED,SAAS,WAAW;;IACnB,MAAM,MAAM,GAAG,MAAA,gBAAM,CAAC,MAAM,0CAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAgB,cAAc,CAAC,KAAY;;IAC1C,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,0CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WACrC,OAAA,MAAA,GAAG,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA,EAAA,CAC7C,CAAC;IAEF,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,KAAK,GAAG,EAAyB,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC;YAC/C,KAAK,CAAC,MAAM,CAAC,GAAG,GAAI,CAAC,CAAC,CAAC,CAAC;SACxB;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACpB;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AApBD,wCAoBC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 452a9151..0a304281 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -7,10 +7,15 @@ import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; import { - deidentifyWithInfoTypeTransformations, - deidentifyWithRecordTransformations, + deidentifyWithInfoTypeTransformations, + deidentifyWithRecordTransformations, } from "./deidentify"; -import { MaskTransformation, RedactTransformation } from "./transofmrations"; +import { + MaskTransformation, + RedactTransformation, + ReplaceTransformation, + ReplaceWithInfoTypeTransformation, +} from "./transofmrations"; import { reidentifyWithInfoTypeTransformations } from "./reidentify"; admin.initializeApp(); @@ -20,135 +25,140 @@ const bigqueryClient = new BigQuery(); const bigqueryConnectionClient = new ConnectionServiceClient(); exports.deidentifyData = functions.https.onRequest( - async (request, response) => { - const { calls, userDefinedContext } = request.body; - - functions.logger.debug("Incoming request from BigQuery", calls); - var transformation; - - switch (config.technique) { - case "redact": - transformation = new RedactTransformation(); - break; - - default: - transformation = new MaskTransformation(); - } - - try { - switch (userDefinedContext.method) { - case "INFO_TYPE": - response.send({ - replies: await deidentifyWithInfoTypeTransformations( - calls, - dlp, - transformation - ), - }); - break; - case "RECORD": - response.send({ - replies: await deidentifyWithRecordTransformations( - calls, - dlp, - transformation - ), - }); - break; - default: - response.status(400).send({ errorMessage: "Invalid method" }); - break; - } - } catch (error) { - functions.logger.error(error); - - response.status(400).send({ errorMessage: error }); - } - } + async (request, response) => { + const { calls, userDefinedContext } = request.body; + + functions.logger.debug("Incoming request from BigQuery", calls); + var transformation; + + switch (config.technique) { + case "redact": + transformation = new RedactTransformation(); + break; + case "fixed": + transformation = new ReplaceTransformation(); + break; + case "replaceWithInfoType": + transformation = new ReplaceWithInfoTypeTransformation(); + break; + default: + transformation = new MaskTransformation(); + } + + try { + switch (userDefinedContext.method) { + case "INFO_TYPE": + response.send({ + replies: await deidentifyWithInfoTypeTransformations( + calls, + dlp, + transformation + ), + }); + break; + case "RECORD": + response.send({ + replies: await deidentifyWithRecordTransformations( + calls, + dlp, + transformation + ), + }); + break; + default: + response.status(400).send({ errorMessage: "Invalid method" }); + break; + } + } catch (error) { + functions.logger.error(error); + + response.status(400).send({ errorMessage: error }); + } + } ); exports.reidentifyData = functions.https.onRequest( - async (request, response) => { - const { calls, userDefinedContext } = request.body; - - functions.logger.debug("Incoming request from BigQuery", calls); - - var transformation; - - switch (config.technique) { - default: - response.status(400).send("Invalid or irreversable technique"); - return; - } - - try { - if (userDefinedContext.method === "INFO_TYPE") { - response.send({ - replies: await reidentifyWithInfoTypeTransformations( - calls, - dlp, - transformation - ), - }); - } else if (userDefinedContext.method === "RECORD") { - response.send({ - replies: await deidentifyWithRecordTransformations( - calls, - dlp, - transformation - ), - }); - } else { - response.status(400).send("Invalid method"); - } - } catch (error) { - functions.logger.error(error); - response.status(400).send({ errorMessage: error }); - } - } + async (request, response) => { + const { calls, userDefinedContext } = request.body; + + functions.logger.debug("Incoming request from BigQuery", calls); + + var transformation; + + switch (config.technique) { + default: + response.status(400).send("Invalid or irreversable technique"); + return; + } + + try { + if (userDefinedContext.method === "INFO_TYPE") { + response.send({ + replies: await reidentifyWithInfoTypeTransformations( + calls, + dlp, + transformation + ), + }); + } else if (userDefinedContext.method === "RECORD") { + response.send({ + replies: await deidentifyWithRecordTransformations( + calls, + dlp, + transformation + ), + }); + } else { + response.status(400).send("Invalid method"); + } + } catch (error) { + functions.logger.error(error); + response.status(400).send({ errorMessage: error }); + } + } ); exports.createBigQueryConnection = functions.tasks - .taskQueue() - .onDispatch(async () => { - const runtime = getExtensions().runtime(); - - const parent = `projects/${config.projectId}/locations/${config.location}`; - const instanceId = "ext-" + config.extInstanceId; - var connection; - - try { - connection = await bigqueryConnectionClient.createConnection({ - parent: parent, - connectionId: instanceId, - connection: { - cloudResource: { - serviceAccountId: `${instanceId}@${config.projectId}.iam.gserviceaccount.com`, - }, - name: instanceId, - friendlyName: "DLP Extension", - }, - }); - - functions.logger.info("Connection successfully created 🎉", connection); - } catch (error: any) { - if (error["code"] === 6) { - functions.logger.warn( - `Connection ${instanceId} already exists, will continue creating functions` - ); - } else { - functions.logger.error(error); - await runtime.setProcessingState( - "PROCESSING_FAILED", - "Error creating connection. Check logs for more details." - ); - - return; - } - } - - try { - const query = ` + .taskQueue() + .onDispatch(async () => { + const runtime = getExtensions().runtime(); + + const parent = `projects/${config.projectId}/locations/${config.location}`; + const instanceId = "ext-" + config.extInstanceId; + var connection; + + try { + connection = await bigqueryConnectionClient.createConnection({ + parent: parent, + connectionId: instanceId, + connection: { + cloudResource: { + serviceAccountId: `${instanceId}@${config.projectId}.iam.gserviceaccount.com`, + }, + name: instanceId, + friendlyName: "DLP Extension", + }, + }); + + functions.logger.info("Connection successfully created 🎉", connection); + } catch (error: any) { + if (error["code"] === 6) { + functions.logger.warn( + `Connection ${instanceId} already exists, will continue creating functions` + ); + } else { + functions.logger.error(error); + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Error creating connection. Check logs for more details." + ); + + return; + } + } + + try { + const query = ` BEGIN CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${instanceId}\` @@ -165,34 +175,34 @@ exports.createBigQueryConnection = functions.tasks END; `; - const options = { - query: query, - location: config.location, - }; - - // Run the query as a job - const [job] = await bigqueryClient.createQueryJob(options); - functions.logger.debug(`Job ${job.id} started.`); - - // Wait for the query to finish - await job.getQueryResults(); - - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - "Connections created successfully." - ); - } catch (error: any) { - if (error["code"] === 6) { - functions.logger.warn(`Functions already exists.`); - } else { - functions.logger.error(error); - - await runtime.setProcessingState( - "PROCESSING_FAILED", - "Connections were not created, check logs for more details." - ); - - return; - } - } - }); + const options = { + query: query, + location: config.location, + }; + + // Run the query as a job + const [job] = await bigqueryClient.createQueryJob(options); + functions.logger.debug(`Job ${job.id} started.`); + + // Wait for the query to finish + await job.getQueryResults(); + + await runtime.setProcessingState( + "PROCESSING_COMPLETE", + "Connections created successfully." + ); + } catch (error: any) { + if (error["code"] === 6) { + functions.logger.warn(`Functions already exists.`); + } else { + functions.logger.error(error); + + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Connections were not created, check logs for more details." + ); + + return; + } + } + }); diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts index a48f4dfe..e4704172 100644 --- a/bigquery-dlp-function/functions/src/transofmrations.ts +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -8,165 +8,255 @@ type Row = protos.google.privacy.dlp.v2.Table.Row; type Table = protos.google.privacy.dlp.v2.ITable | undefined | null; class Transformation { - parent: string; - deidentifyConfig: DeidentifyRequest = {}; - reidentifyConfig: ReidentifyRequest = {}; + parent: string; + deidentifyConfig: DeidentifyRequest = {}; + reidentifyConfig: ReidentifyRequest = {}; - constructor() { - this.parent = `projects/${config.projectId}/locations/${config.location}`; - } + constructor() { + this.parent = `projects/${config.projectId}/locations/${config.location}`; + } } export class MaskTransformation extends Transformation { - /** - * Replace a value by a mask character. - * - * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. - * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. - */ - constructor(mask?: string, numberToMask?: number) { - super(); - const maskingConfig = { - ...(config.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, - }, - }, - }, - ], - }, - }), - ...(config.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, - }, - }, - }, - ], - }, - }), - }; - - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: maskingConfig, - }; - } + /** + * Replace a value by a mask character. + * + * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. + * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. + */ + constructor(mask?: string, numberToMask?: number) { + super(); + const maskingConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }), + }; + + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: maskingConfig, + }; + } } export class RedactTransformation extends Transformation { - /** - * Redacts a value by removing it. - */ - constructor() { - super(); - const redactConfig = { - ...(config.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - redactConfig: {}, - }, - }, - ], - }, - }), - ...(config.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - redactConfig: {}, - }, - }, - ], - }, - }), - }; - - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: redactConfig, - }; - } + /** + * Redacts a value by removing it. + */ + constructor() { + super(); + const redactConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }), + }; + + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: redactConfig, + }; + } +} + +export class ReplaceTransformation extends Transformation { + /** + * Replace with a specified value. + */ + constructor() { + super(); + const _replaceConfig = { + newValue: { + // TODO make configurable? + stringValue: "REPLACED", + }, + }; + + const replaceConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + replaceConfig: _replaceConfig, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + replaceConfig: _replaceConfig, + }, + }, + ], + }, + }), + }; + + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: replaceConfig, + }; + } +} + +export class ReplaceWithInfoTypeTransformation extends Transformation { + /** + * Replace with a specified value. + */ + constructor() { + super(); + const _replaceConfig = { + // TODO make configurable? + partToExtract: "MONTH", + }; + + const replaceConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + replaceWithInfoTypeConfig: _replaceConfig, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + replaceWithInfoTypeConfig: _replaceConfig, + }, + }, + ], + }, + }), + }; + + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: replaceConfig, + }; + } } export function rowsToTable(rows: []) { - let table = { - headers: [] as FieldId[], - rows: [] as Row[], - }; - - for (const row of rows) { - const data = row[0] as Record; - - const keys = Object.keys(data); - const values = Object.values(data); - - if (table.headers.length === 0) { - // Add headers only once - table.headers = keys.map((key) => { - const field = protos.google.privacy.dlp.v2.FieldId.create({ - name: key, - }); - return field; - }); - } - - const tableRow = protos.google.privacy.dlp.v2.Table.Row.create({ - values: values.map((v) => { - const field = protos.google.privacy.dlp.v2.Value.create({ - stringValue: v, - }); - return field; - }), - }); - - table.rows.push(tableRow); - } - - return table; + let table = { + headers: [] as FieldId[], + rows: [] as Row[], + }; + + for (const row of rows) { + const data = row[0] as Record; + + const keys = Object.keys(data); + const values = Object.values(data); + + if (table.headers.length === 0) { + // Add headers only once + table.headers = keys.map((key) => { + const field = protos.google.privacy.dlp.v2.FieldId.create({ + name: key, + }); + return field; + }); + } + + const tableRow = protos.google.privacy.dlp.v2.Table.Row.create({ + values: values.map((v) => { + const field = protos.google.privacy.dlp.v2.Value.create({ + stringValue: v, + }); + return field; + }), + }); + + table.rows.push(tableRow); + } + + return table; } -export function getFieldIds() { - const fields = config.fields?.split(","); - const fieldIds = fields?.map((field) => { - return { name: field }; - }); - return fieldIds; +function getFieldIds() { + const fields = config.fields?.split(","); + const fieldIds = fields?.map((field) => { + return { name: field }; + }); + return fieldIds; } export function tableToReplies(table: Table) { - const replies = []; - const rows = table?.rows?.map((row) => - row.values?.map((value) => value.stringValue) - ); + const replies = []; + const rows = table?.rows?.map((row) => + row.values?.map((value) => value.stringValue) + ); - if (!rows || !table || !table.headers) return []; + if (!rows || !table || !table.headers) return []; - for (const row of rows) { - const reply = {} as Record; + for (const row of rows) { + const reply = {} as Record; - for (let i = 0; i < table.headers.length; i++) { - const header = table.headers[i].name as string; - reply[header] = row![i]; - } + for (let i = 0; i < table.headers.length; i++) { + const header = table.headers[i].name as string; + reply[header] = row![i]; + } - replies.push(reply); - } + replies.push(reply); + } - return replies; + return replies; } From 7472bf4e4a2e013247783063170173ef46868b31 Mon Sep 17 00:00:00 2001 From: Mais Date: Wed, 1 Mar 2023 14:29:09 +0300 Subject: [PATCH 32/37] fix: use the method declared in config --- bigquery-dlp-function/functions/lib/config.js | 3 +- .../functions/lib/config.js.map | 2 +- .../functions/lib/deidentify.js | 4 +- .../functions/lib/deidentify.js.map | 2 +- bigquery-dlp-function/functions/lib/index.js | 16 ++- .../functions/lib/index.js.map | 2 +- .../functions/lib/transofmrations.js | 3 +- .../functions/lib/transofmrations.js.map | 2 +- bigquery-dlp-function/functions/src/config.ts | 14 +-- .../functions/src/deidentify.ts | 106 ++++++++++-------- bigquery-dlp-function/functions/src/index.ts | 16 ++- .../functions/src/transofmrations.ts | 3 +- 12 files changed, 89 insertions(+), 84 deletions(-) diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js index 1a8a9315..baf0e8a9 100644 --- a/bigquery-dlp-function/functions/lib/config.js +++ b/bigquery-dlp-function/functions/lib/config.js @@ -14,6 +14,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ +var _a; Object.defineProperty(exports, "__esModule", { value: true }); exports.default = { projectId: process.env.PROJECT_ID, @@ -22,6 +23,6 @@ exports.default = { location: process.env.LOCATION, method: process.env.TRANSFORMATION_METHOD, technique: process.env.TRANSFORMATION_TECHNIQUE, - fields: process.env.FIELDS_TO_TRANSFORM, + fields: (_a = process.env.FIELDS_TO_TRANSFORM) === null || _a === void 0 ? void 0 : _a.split(","), }; //# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map index 4b7871cf..cae89e2b 100644 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ b/bigquery-dlp-function/functions/lib/config.js.map @@ -1 +1 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;AAEH,kBAAe;IACb,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;IACzC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;IAC/C,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,mBAAmB;CACxC,CAAC"} \ No newline at end of file +{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,kBAAe;IACd,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;IACzC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;IAC/C,MAAM,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,mBAAmB,0CAAE,KAAK,CAAC,GAAG,CAAC;CACnD,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js b/bigquery-dlp-function/functions/lib/deidentify.js index 9c5a52a8..dc7e9b74 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js +++ b/bigquery-dlp-function/functions/lib/deidentify.js @@ -9,7 +9,7 @@ const transofmrations_1 = require("./transofmrations"); * * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#infotype_transformations * - * @param {string} text The text to deidentify. + * @param {Array} rows The text to deidentify. * * @returns {Promise>} The deidentified record. */ @@ -37,7 +37,7 @@ exports.deidentifyWithInfoTypeTransformations = deidentifyWithInfoTypeTransforma * * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#record_transformations * - * @param {string} text The text to deidentify. + * @param {Array} rows The rows with data to deidentify. * * @returns {Promise} The deidentified text. */ diff --git a/bigquery-dlp-function/functions/lib/deidentify.js.map b/bigquery-dlp-function/functions/lib/deidentify.js.map index 55f4380a..c52659bc 100644 --- a/bigquery-dlp-function/functions/lib/deidentify.js.map +++ b/bigquery-dlp-function/functions/lib/deidentify.js.map @@ -1 +1 @@ -{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,uDAK2B;AAE3B;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACzB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AA3BD,sFA2BC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mCAAmC,CACvD,IAAS,EACT,MAAwB,EACxB,cAAyD;;IAEzD,IAAI,KAAK,CAAC;IAEV,IAAI;QACF,iCAAiC;QACjC,KAAK,GAAG,IAAA,6BAAW,EAAC,IAAI,CAAC,CAAC;KAC3B;IAAC,OAAO,KAAK,EAAE;QACd,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;KACb;IAED,sCAAsC;IACtC,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE;YACJ,KAAK,EAAE,KAAK;SACb,GACF,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC,CAAC;IAE7D,OAAO,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;AAC9C,CAAC;AA7BD,kFA6BC"} \ No newline at end of file +{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,uDAO2B;AAE3B;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CAC1D,IAAQ,EACR,MAAwB,EACxB,cAIoC;;IAEpC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACT,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACxB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aACjC;SACD;QAED,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC7B;IAED,OAAO,iBAAiB,CAAC;AAC1B,CAAC;AA/BD,sFA+BC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mCAAmC,CACxD,IAAQ,EACR,MAAwB,EACxB,cAIoC;;IAEpC,IAAI,KAAK,CAAC;IAEV,IAAI;QACH,iCAAiC;QACjC,KAAK,GAAG,IAAA,6BAAW,EAAC,IAAI,CAAC,CAAC;KAC1B;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;KACZ;IAED,sCAAsC;IACtC,MAAM,OAAO,mCACT,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE;YACL,KAAK,EAAE,KAAK;SACZ,GACD,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC,CAAC;IAE7D,OAAO,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAjCD,kFAiCC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js index 6c0a8be1..77cd5d45 100644 --- a/bigquery-dlp-function/functions/lib/index.js +++ b/bigquery-dlp-function/functions/lib/index.js @@ -15,7 +15,7 @@ const dlp = new dlp_1.default.DlpServiceClient(); const bigqueryClient = new bigquery_1.BigQuery(); const bigqueryConnectionClient = new bigquery_connection_1.ConnectionServiceClient(); exports.deidentifyData = functions.https.onRequest(async (request, response) => { - const { calls, userDefinedContext } = request.body; + const { calls } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); var transformation; switch (config_1.default.technique) { @@ -32,7 +32,7 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => transformation = new transofmrations_1.MaskTransformation(); } try { - switch (userDefinedContext.method) { + switch (config_1.default.method) { case "INFO_TYPE": response.send({ replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, transformation), @@ -54,7 +54,7 @@ exports.deidentifyData = functions.https.onRequest(async (request, response) => } }); exports.reidentifyData = functions.https.onRequest(async (request, response) => { - const { calls, userDefinedContext } = request.body; + const { calls } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); var transformation; switch (config_1.default.technique) { @@ -63,12 +63,12 @@ exports.reidentifyData = functions.https.onRequest(async (request, response) => return; } try { - if (userDefinedContext.method === "INFO_TYPE") { + if (config_1.default.method === "INFO_TYPE") { response.send({ replies: await (0, reidentify_1.reidentifyWithInfoTypeTransformations)(calls, dlp, transformation), }); } - else if (userDefinedContext.method === "RECORD") { + else if (config_1.default.method === "RECORD") { response.send({ replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp, transformation), }); @@ -119,14 +119,12 @@ exports.createBigQueryConnection = functions.tasks CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-deidentifyData', - user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-deidentifyData' ); CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-reidentifyData', - user_defined_context = [("method", "${config_1.default.method}"), ("technique", "${config_1.default.technique}")] + endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-reidentifyData' ); END; `; diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map index acc6a65f..27031d7a 100644 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ b/bigquery-dlp-function/functions/lib/index.js.map @@ -1 +1 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AACtB,uDAK2B;AAC3B,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB,KAAK,QAAQ;YACZ,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;YAC5C,MAAM;QACP,KAAK,OAAO;YACX,cAAc,GAAG,IAAI,uCAAqB,EAAE,CAAC;YAC7C,MAAM;QACP,KAAK,qBAAqB;YACzB,cAAc,GAAG,IAAI,mDAAiC,EAAE,CAAC;YACzD,MAAM;QACP;YACC,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;KAC3C;IAED,IAAI;QACH,QAAQ,kBAAkB,CAAC,MAAM,EAAE;YAClC,KAAK,WAAW;gBACf,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP,KAAK,QAAQ;gBACZ,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP;gBACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC9D,MAAM;SACP;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,kBAAkB,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAEnD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB;YACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC/D,OAAO;KACR;IAED,IAAI;QACH,IAAI,kBAAkB,CAAC,MAAM,KAAK,WAAW,EAAE;YAC9C,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM,IAAI,kBAAkB,CAAC,MAAM,KAAK,QAAQ,EAAE;YAClD,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM;YACN,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC5C;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAChD,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACtB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACH,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC5D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACX,aAAa,EAAE;oBACd,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC7E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC7B;SACD,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACxE;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CACpB,cAAc,UAAU,mDAAmD,CAC3E,CAAC;SACF;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,yDAAyD,CACzD,CAAC;YAEF,OAAO;SACP;KACD;IAED,IAAI;QACH,MAAM,KAAK,GAAG;;8BAEa,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;8BAEvE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;kDACpD,gBAAM,CAAC,MAAM,sBAAsB,gBAAM,CAAC,SAAS;;;UAG3F,CAAC;QAER,MAAM,OAAO,GAAG;YACf,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SACzB,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,qBAAqB,EACrB,mCAAmC,CACnC,CAAC;KACF;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACnD;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,4DAA4D,CAC5D,CAAC;YAEF,OAAO;SACP;KACD;AACF,CAAC,CAAC,CAAC"} \ No newline at end of file +{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AACtB,uDAK2B;AAC3B,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB,KAAK,QAAQ;YACZ,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;YAC5C,MAAM;QACP,KAAK,OAAO;YACX,cAAc,GAAG,IAAI,uCAAqB,EAAE,CAAC;YAC7C,MAAM;QACP,KAAK,qBAAqB;YACzB,cAAc,GAAG,IAAI,mDAAiC,EAAE,CAAC;YACzD,MAAM;QACP;YACC,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;KAC3C;IAED,IAAI;QACH,QAAQ,gBAAM,CAAC,MAAM,EAAE;YACtB,KAAK,WAAW;gBACf,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP,KAAK,QAAQ;gBACZ,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP;gBACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC9D,MAAM;SACP;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB;YACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC/D,OAAO;KACR;IAED,IAAI;QACH,IAAI,gBAAM,CAAC,MAAM,KAAK,WAAW,EAAE;YAClC,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM,IAAI,gBAAM,CAAC,MAAM,KAAK,QAAQ,EAAE;YACtC,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM;YACN,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC5C;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAChD,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACtB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACH,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC5D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACX,aAAa,EAAE;oBACd,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC7E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC7B;SACD,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACxE;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CACpB,cAAc,UAAU,mDAAmD,CAC3E,CAAC;SACF;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,yDAAyD,CACzD,CAAC;YAEF,OAAO;SACP;KACD;IAED,IAAI;QACH,MAAM,KAAK,GAAG;;8BAEa,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;;8BAExE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;;;UAG5F,CAAC;QAER,MAAM,OAAO,GAAG;YACf,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SACzB,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,qBAAqB,EACrB,mCAAmC,CACnC,CAAC;KACF;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACnD;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,4DAA4D,CAC5D,CAAC;YAEF,OAAO;SACP;KACD;AACF,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js index 4ec90c82..b5ca74b3 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js +++ b/bigquery-dlp-function/functions/lib/transofmrations.js @@ -202,8 +202,7 @@ function rowsToTable(rows) { exports.rowsToTable = rowsToTable; function getFieldIds() { var _a; - const fields = (_a = config_1.default.fields) === null || _a === void 0 ? void 0 : _a.split(","); - const fieldIds = fields === null || fields === void 0 ? void 0 : fields.map((field) => { + const fieldIds = (_a = config_1.default.fields) === null || _a === void 0 ? void 0 : _a.map((field) => { return { name: field }; }); return fieldIds; diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js.map b/bigquery-dlp-function/functions/lib/transofmrations.js.map index 6dc95ece..8a845575 100644 --- a/bigquery-dlp-function/functions/lib/transofmrations.js.map +++ b/bigquery-dlp-function/functions/lib/transofmrations.js.map @@ -1 +1 @@ -{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAC9B,2CAA2C;AAQ3C,MAAM,cAAc;IAKnB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGxC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,CAAC;CACD;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACrD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC/C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA9CD,gDA8CC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACvD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,YAAY,mCACd,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,YAAY;SAC9B,CAAC;IACH,CAAC;CACD;AArCD,oDAqCC;AAED,MAAa,qBAAsB,SAAQ,cAAc;IACxD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,QAAQ,EAAE;gBACT,0BAA0B;gBAC1B,WAAW,EAAE,UAAU;aACvB;SACD,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA5CD,sDA4CC;AAED,MAAa,iCAAkC,SAAQ,cAAc;IACpE;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,0BAA0B;YAC1B,aAAa,EAAE,OAAO;SACtB,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA1CD,8EA0CC;AAED,SAAgB,WAAW,CAAC,IAAQ;IACnC,IAAI,KAAK,GAAG;QACX,OAAO,EAAE,EAAe;QACxB,IAAI,EAAE,EAAW;KACjB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/B,wBAAwB;YACxB,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAChC,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;oBACzD,IAAI,EAAE,GAAG;iBACT,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAC9D,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxB,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;oBACvD,WAAW,EAAE,CAAC;iBACd,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC;SACF,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC1B;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAnCD,kCAmCC;AAED,SAAS,WAAW;;IACnB,MAAM,MAAM,GAAG,MAAA,gBAAM,CAAC,MAAM,0CAAE,KAAK,CAAC,GAAG,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,aAAN,MAAM,uBAAN,MAAM,CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QACtC,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAgB,cAAc,CAAC,KAAY;;IAC1C,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,0CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WACrC,OAAA,MAAA,GAAG,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA,EAAA,CAC7C,CAAC;IAEF,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,KAAK,GAAG,EAAyB,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC;YAC/C,KAAK,CAAC,MAAM,CAAC,GAAG,GAAI,CAAC,CAAC,CAAC,CAAC;SACxB;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACpB;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AApBD,wCAoBC"} \ No newline at end of file +{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAC9B,2CAA2C;AAQ3C,MAAM,cAAc;IAKnB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGxC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,CAAC;CACD;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACrD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC/C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA9CD,gDA8CC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACvD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,YAAY,mCACd,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,YAAY;SAC9B,CAAC;IACH,CAAC;CACD;AArCD,oDAqCC;AAED,MAAa,qBAAsB,SAAQ,cAAc;IACxD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,QAAQ,EAAE;gBACT,0BAA0B;gBAC1B,WAAW,EAAE,UAAU;aACvB;SACD,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA5CD,sDA4CC;AAED,MAAa,iCAAkC,SAAQ,cAAc;IACpE;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,0BAA0B;YAC1B,aAAa,EAAE,OAAO;SACtB,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA1CD,8EA0CC;AAED,SAAgB,WAAW,CAAC,IAAQ;IACnC,IAAI,KAAK,GAAG;QACX,OAAO,EAAE,EAAe;QACxB,IAAI,EAAE,EAAW;KACjB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/B,wBAAwB;YACxB,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAChC,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;oBACzD,IAAI,EAAE,GAAG;iBACT,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAC9D,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxB,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;oBACvD,WAAW,EAAE,CAAC;iBACd,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC;SACF,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC1B;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAnCD,kCAmCC;AAED,SAAS,WAAW;;IACnB,MAAM,QAAQ,GAAG,MAAA,gBAAM,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAgB,cAAc,CAAC,KAAY;;IAC1C,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,0CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WACrC,OAAA,MAAA,GAAG,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA,EAAA,CAC7C,CAAC;IAEF,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,KAAK,GAAG,EAAyB,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC;YAC/C,KAAK,CAAC,MAAM,CAAC,GAAG,GAAI,CAAC,CAAC,CAAC,CAAC;SACxB;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACpB;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AApBD,wCAoBC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 54399b00..4b84d99f 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -15,11 +15,11 @@ */ export default { - projectId: process.env.PROJECT_ID, - extInstanceId: process.env.EXT_INSTANCE_ID, - datasetId: process.env.DATASET_ID, - location: process.env.LOCATION, - method: process.env.TRANSFORMATION_METHOD, - technique: process.env.TRANSFORMATION_TECHNIQUE, - fields: process.env.FIELDS_TO_TRANSFORM, + projectId: process.env.PROJECT_ID, + extInstanceId: process.env.EXT_INSTANCE_ID, + datasetId: process.env.DATASET_ID, + location: process.env.LOCATION, + method: process.env.TRANSFORMATION_METHOD, + technique: process.env.TRANSFORMATION_TECHNIQUE, + fields: process.env.FIELDS_TO_TRANSFORM?.split(","), }; diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index acdb9897..e72a6ffd 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -2,10 +2,12 @@ import * as functions from "firebase-functions"; import { DlpServiceClient } from "@google-cloud/dlp"; import { - MaskTransformation, - RedactTransformation, - rowsToTable, - tableToReplies, + MaskTransformation, + RedactTransformation, + ReplaceTransformation, + ReplaceWithInfoTypeTransformation, + rowsToTable, + tableToReplies, } from "./transofmrations"; /** @@ -14,37 +16,41 @@ import { * * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#infotype_transformations * - * @param {string} text The text to deidentify. + * @param {Array} rows The text to deidentify. * * @returns {Promise>} The deidentified record. */ export async function deidentifyWithInfoTypeTransformations( - rows: [], - client: DlpServiceClient, - transformation: MaskTransformation | RedactTransformation + rows: [], + client: DlpServiceClient, + transformation: + | MaskTransformation + | RedactTransformation + | ReplaceTransformation + | ReplaceWithInfoTypeTransformation ): Promise> { - const deidentifiedItems = []; + const deidentifiedItems = []; - for (const row of rows) { - const data = row[0] as Record; + for (const row of rows) { + const data = row[0] as Record; - for (const key in data) { - if (data.hasOwnProperty(key)) { - const element = data[key]; - const request = { - ...transformation.deidentifyConfig, - item: { value: element }, - }; + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = { + ...transformation.deidentifyConfig, + item: { value: element }, + }; - const [response] = await client.deidentifyContent(request); - data[key] = response.item?.value; - } - } + const [response] = await client.deidentifyContent(request); + data[key] = response.item?.value; + } + } - deidentifiedItems.push(data); - } + deidentifiedItems.push(data); + } - return deidentifiedItems; + return deidentifiedItems; } /** @@ -53,37 +59,41 @@ export async function deidentifyWithInfoTypeTransformations( * * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#record_transformations * - * @param {string} text The text to deidentify. + * @param {Array} rows The rows with data to deidentify. * * @returns {Promise} The deidentified text. */ export async function deidentifyWithRecordTransformations( - rows: any, - client: DlpServiceClient, - transformation: MaskTransformation | RedactTransformation + rows: [], + client: DlpServiceClient, + transformation: + | MaskTransformation + | RedactTransformation + | ReplaceTransformation + | ReplaceWithInfoTypeTransformation ) { - let table; + let table; - try { - // Convert raw rows to Table type - table = rowsToTable(rows); - } catch (error) { - functions.logger.debug(`Error converting rows to Table type.`); - throw error; - } + try { + // Convert raw rows to Table type + table = rowsToTable(rows); + } catch (error) { + functions.logger.debug(`Error converting rows to Table type.`); + throw error; + } - // Construct de-identification request - const request = { - ...transformation.deidentifyConfig, - item: { - table: table, - }, - }; + // Construct de-identification request + const request = { + ...transformation.deidentifyConfig, + item: { + table: table, + }, + }; - // Run deidentification request - const [response] = await client.deidentifyContent(request); + // Run deidentification request + const [response] = await client.deidentifyContent(request); - functions.logger.debug(tableToReplies(response.item?.table)); + functions.logger.debug(tableToReplies(response.item?.table)); - return tableToReplies(response.item?.table); + return tableToReplies(response.item?.table); } diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 0a304281..4d9c0333 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -26,7 +26,7 @@ const bigqueryConnectionClient = new ConnectionServiceClient(); exports.deidentifyData = functions.https.onRequest( async (request, response) => { - const { calls, userDefinedContext } = request.body; + const { calls } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); var transformation; @@ -46,7 +46,7 @@ exports.deidentifyData = functions.https.onRequest( } try { - switch (userDefinedContext.method) { + switch (config.method) { case "INFO_TYPE": response.send({ replies: await deidentifyWithInfoTypeTransformations( @@ -79,7 +79,7 @@ exports.deidentifyData = functions.https.onRequest( exports.reidentifyData = functions.https.onRequest( async (request, response) => { - const { calls, userDefinedContext } = request.body; + const { calls } = request.body; functions.logger.debug("Incoming request from BigQuery", calls); @@ -92,7 +92,7 @@ exports.reidentifyData = functions.https.onRequest( } try { - if (userDefinedContext.method === "INFO_TYPE") { + if (config.method === "INFO_TYPE") { response.send({ replies: await reidentifyWithInfoTypeTransformations( calls, @@ -100,7 +100,7 @@ exports.reidentifyData = functions.https.onRequest( transformation ), }); - } else if (userDefinedContext.method === "RECORD") { + } else if (config.method === "RECORD") { response.send({ replies: await deidentifyWithRecordTransformations( calls, @@ -163,14 +163,12 @@ exports.createBigQueryConnection = functions.tasks CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/${instanceId}-deidentifyData', - user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/${instanceId}-deidentifyData' ); CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.reidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${instanceId}\` OPTIONS ( - endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/${instanceId}-reidentifyData', - user_defined_context = [("method", "${config.method}"), ("technique", "${config.technique}")] + endpoint = 'https://${config.location}-${config.projectId}.cloudfunctions.net/${instanceId}-reidentifyData' ); END; `; diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts index e4704172..49e4ae2a 100644 --- a/bigquery-dlp-function/functions/src/transofmrations.ts +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -232,8 +232,7 @@ export function rowsToTable(rows: []) { } function getFieldIds() { - const fields = config.fields?.split(","); - const fieldIds = fields?.map((field) => { + const fieldIds = config.fields?.map((field) => { return { name: field }; }); return fieldIds; From 4dbf8930f6f45f7835da35be823bd16d091db88e Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Fri, 7 Apr 2023 17:49:51 +0100 Subject: [PATCH 33/37] chore: added gitignore --- bigquery-dlp-function/functions/.gitignore | 8 + bigquery-dlp-function/functions/lib/config.js | 28 --- .../functions/lib/config.js.map | 1 - .../functions/lib/deidentify.js | 65 ----- .../functions/lib/deidentify.js.map | 1 - bigquery-dlp-function/functions/lib/index.js | 153 ------------ .../functions/lib/index.js.map | 1 - .../functions/lib/reidentify.js | 32 --- .../functions/lib/reidentify.js.map | 1 - .../functions/lib/transofmrations.js | 227 ------------------ .../functions/lib/transofmrations.js.map | 1 - .../functions/lib/types/call_mode.js | 9 - .../functions/lib/types/call_mode.js.map | 1 - 13 files changed, 8 insertions(+), 520 deletions(-) create mode 100644 bigquery-dlp-function/functions/.gitignore delete mode 100644 bigquery-dlp-function/functions/lib/config.js delete mode 100644 bigquery-dlp-function/functions/lib/config.js.map delete mode 100644 bigquery-dlp-function/functions/lib/deidentify.js delete mode 100644 bigquery-dlp-function/functions/lib/deidentify.js.map delete mode 100644 bigquery-dlp-function/functions/lib/index.js delete mode 100644 bigquery-dlp-function/functions/lib/index.js.map delete mode 100644 bigquery-dlp-function/functions/lib/reidentify.js delete mode 100644 bigquery-dlp-function/functions/lib/reidentify.js.map delete mode 100644 bigquery-dlp-function/functions/lib/transofmrations.js delete mode 100644 bigquery-dlp-function/functions/lib/transofmrations.js.map delete mode 100644 bigquery-dlp-function/functions/lib/types/call_mode.js delete mode 100644 bigquery-dlp-function/functions/lib/types/call_mode.js.map diff --git a/bigquery-dlp-function/functions/.gitignore b/bigquery-dlp-function/functions/.gitignore new file mode 100644 index 00000000..7fbb8b40 --- /dev/null +++ b/bigquery-dlp-function/functions/.gitignore @@ -0,0 +1,8 @@ +## Compiled JavaScript files +**/*.js +**/*.js.map + +# Typescript v1 declaration files +typings/ + +node_modules/ \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js b/bigquery-dlp-function/functions/lib/config.js deleted file mode 100644 index baf0e8a9..00000000 --- a/bigquery-dlp-function/functions/lib/config.js +++ /dev/null @@ -1,28 +0,0 @@ -"use strict"; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -var _a; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.default = { - projectId: process.env.PROJECT_ID, - extInstanceId: process.env.EXT_INSTANCE_ID, - datasetId: process.env.DATASET_ID, - location: process.env.LOCATION, - method: process.env.TRANSFORMATION_METHOD, - technique: process.env.TRANSFORMATION_TECHNIQUE, - fields: (_a = process.env.FIELDS_TO_TRANSFORM) === null || _a === void 0 ? void 0 : _a.split(","), -}; -//# sourceMappingURL=config.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/config.js.map b/bigquery-dlp-function/functions/lib/config.js.map deleted file mode 100644 index cae89e2b..00000000 --- a/bigquery-dlp-function/functions/lib/config.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"config.js","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":";AAAA;;;;;;;;;;;;;;GAcG;;;AAEH,kBAAe;IACd,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,aAAa,EAAE,OAAO,CAAC,GAAG,CAAC,eAAe;IAC1C,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,UAAU;IACjC,QAAQ,EAAE,OAAO,CAAC,GAAG,CAAC,QAAQ;IAC9B,MAAM,EAAE,OAAO,CAAC,GAAG,CAAC,qBAAqB;IACzC,SAAS,EAAE,OAAO,CAAC,GAAG,CAAC,wBAAwB;IAC/C,MAAM,EAAE,MAAA,OAAO,CAAC,GAAG,CAAC,mBAAmB,0CAAE,KAAK,CAAC,GAAG,CAAC;CACnD,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js b/bigquery-dlp-function/functions/lib/deidentify.js deleted file mode 100644 index dc7e9b74..00000000 --- a/bigquery-dlp-function/functions/lib/deidentify.js +++ /dev/null @@ -1,65 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.deidentifyWithRecordTransformations = exports.deidentifyWithInfoTypeTransformations = void 0; -const functions = require("firebase-functions"); -const transofmrations_1 = require("./transofmrations"); -/** - * Deidentify sensitive data in a string with [the Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) - * using `infoTypeTransformations` method. - * - * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#infotype_transformations - * - * @param {Array} rows The text to deidentify. - * - * @returns {Promise>} The deidentified record. - */ -async function deidentifyWithInfoTypeTransformations(rows, client, transformation) { - var _a; - const deidentifiedItems = []; - for (const row of rows) { - const data = row[0]; - for (const key in data) { - if (data.hasOwnProperty(key)) { - const element = data[key]; - const request = Object.assign(Object.assign({}, transformation.deidentifyConfig), { item: { value: element } }); - const [response] = await client.deidentifyContent(request); - data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; - } - } - deidentifiedItems.push(data); - } - return deidentifiedItems; -} -exports.deidentifyWithInfoTypeTransformations = deidentifyWithInfoTypeTransformations; -/** - * Deidentify sensitive data in a string with the [Data Loss Prevention API](https://cloud.google.com/architecture/de-identification-re-identification-pii-using-cloud-dlp) - * using `recordTransformations` method. - * - * Read more about this method: https://cloud.google.com/dlp/docs/deidentify-sensitive-data#record_transformations - * - * @param {Array} rows The rows with data to deidentify. - * - * @returns {Promise} The deidentified text. - */ -async function deidentifyWithRecordTransformations(rows, client, transformation) { - var _a, _b; - let table; - try { - // Convert raw rows to Table type - table = (0, transofmrations_1.rowsToTable)(rows); - } - catch (error) { - functions.logger.debug(`Error converting rows to Table type.`); - throw error; - } - // Construct de-identification request - const request = Object.assign(Object.assign({}, transformation.deidentifyConfig), { item: { - table: table, - } }); - // Run deidentification request - const [response] = await client.deidentifyContent(request); - functions.logger.debug((0, transofmrations_1.tableToReplies)((_a = response.item) === null || _a === void 0 ? void 0 : _a.table)); - return (0, transofmrations_1.tableToReplies)((_b = response.item) === null || _b === void 0 ? void 0 : _b.table); -} -exports.deidentifyWithRecordTransformations = deidentifyWithRecordTransformations; -//# sourceMappingURL=deidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/deidentify.js.map b/bigquery-dlp-function/functions/lib/deidentify.js.map deleted file mode 100644 index c52659bc..00000000 --- a/bigquery-dlp-function/functions/lib/deidentify.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"deidentify.js","sourceRoot":"","sources":["../src/deidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,uDAO2B;AAE3B;;;;;;;;;GASG;AACI,KAAK,UAAU,qCAAqC,CAC1D,IAAQ,EACR,MAAwB,EACxB,cAIoC;;IAEpC,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACvB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC7B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACT,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,GACxB,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aACjC;SACD;QAED,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC7B;IAED,OAAO,iBAAiB,CAAC;AAC1B,CAAC;AA/BD,sFA+BC;AAED;;;;;;;;;GASG;AACI,KAAK,UAAU,mCAAmC,CACxD,IAAQ,EACR,MAAwB,EACxB,cAIoC;;IAEpC,IAAI,KAAK,CAAC;IAEV,IAAI;QACH,iCAAiC;QACjC,KAAK,GAAG,IAAA,6BAAW,EAAC,IAAI,CAAC,CAAC;KAC1B;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC/D,MAAM,KAAK,CAAC;KACZ;IAED,sCAAsC;IACtC,MAAM,OAAO,mCACT,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE;YACL,KAAK,EAAE,KAAK;SACZ,GACD,CAAC;IAEF,+BAA+B;IAC/B,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;IAE3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC,CAAC;IAE7D,OAAO,IAAA,gCAAc,EAAC,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC,CAAC;AAC7C,CAAC;AAjCD,kFAiCC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js b/bigquery-dlp-function/functions/lib/index.js deleted file mode 100644 index 77cd5d45..00000000 --- a/bigquery-dlp-function/functions/lib/index.js +++ /dev/null @@ -1,153 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const functions = require("firebase-functions"); -const admin = require("firebase-admin"); -const dlp_1 = require("@google-cloud/dlp"); -const bigquery_connection_1 = require("@google-cloud/bigquery-connection"); -const bigquery_1 = require("@google-cloud/bigquery"); -const extensions_1 = require("firebase-admin/extensions"); -const config_1 = require("./config"); -const deidentify_1 = require("./deidentify"); -const transofmrations_1 = require("./transofmrations"); -const reidentify_1 = require("./reidentify"); -admin.initializeApp(); -const dlp = new dlp_1.default.DlpServiceClient(); -const bigqueryClient = new bigquery_1.BigQuery(); -const bigqueryConnectionClient = new bigquery_connection_1.ConnectionServiceClient(); -exports.deidentifyData = functions.https.onRequest(async (request, response) => { - const { calls } = request.body; - functions.logger.debug("Incoming request from BigQuery", calls); - var transformation; - switch (config_1.default.technique) { - case "redact": - transformation = new transofmrations_1.RedactTransformation(); - break; - case "fixed": - transformation = new transofmrations_1.ReplaceTransformation(); - break; - case "replaceWithInfoType": - transformation = new transofmrations_1.ReplaceWithInfoTypeTransformation(); - break; - default: - transformation = new transofmrations_1.MaskTransformation(); - } - try { - switch (config_1.default.method) { - case "INFO_TYPE": - response.send({ - replies: await (0, deidentify_1.deidentifyWithInfoTypeTransformations)(calls, dlp, transformation), - }); - break; - case "RECORD": - response.send({ - replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp, transformation), - }); - break; - default: - response.status(400).send({ errorMessage: "Invalid method" }); - break; - } - } - catch (error) { - functions.logger.error(error); - response.status(400).send({ errorMessage: error }); - } -}); -exports.reidentifyData = functions.https.onRequest(async (request, response) => { - const { calls } = request.body; - functions.logger.debug("Incoming request from BigQuery", calls); - var transformation; - switch (config_1.default.technique) { - default: - response.status(400).send("Invalid or irreversable technique"); - return; - } - try { - if (config_1.default.method === "INFO_TYPE") { - response.send({ - replies: await (0, reidentify_1.reidentifyWithInfoTypeTransformations)(calls, dlp, transformation), - }); - } - else if (config_1.default.method === "RECORD") { - response.send({ - replies: await (0, deidentify_1.deidentifyWithRecordTransformations)(calls, dlp, transformation), - }); - } - else { - response.status(400).send("Invalid method"); - } - } - catch (error) { - functions.logger.error(error); - response.status(400).send({ errorMessage: error }); - } -}); -exports.createBigQueryConnection = functions.tasks - .taskQueue() - .onDispatch(async () => { - const runtime = (0, extensions_1.getExtensions)().runtime(); - const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - const instanceId = "ext-" + config_1.default.extInstanceId; - var connection; - try { - connection = await bigqueryConnectionClient.createConnection({ - parent: parent, - connectionId: instanceId, - connection: { - cloudResource: { - serviceAccountId: `${instanceId}@${config_1.default.projectId}.iam.gserviceaccount.com`, - }, - name: instanceId, - friendlyName: "DLP Extension", - }, - }); - functions.logger.info("Connection successfully created 🎉", connection); - } - catch (error) { - if (error["code"] === 6) { - functions.logger.warn(`Connection ${instanceId} already exists, will continue creating functions`); - } - else { - functions.logger.error(error); - await runtime.setProcessingState("PROCESSING_FAILED", "Error creating connection. Check logs for more details."); - return; - } - } - try { - const query = ` - BEGIN - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.deidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${instanceId}\` - OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-deidentifyData' - ); - CREATE FUNCTION \`${config_1.default.projectId}.${config_1.default.datasetId}\`.reidentify(data JSON) RETURNS JSON - REMOTE WITH CONNECTION \`${config_1.default.projectId}.${config_1.default.location}.${instanceId}\` - OPTIONS ( - endpoint = 'https://${config_1.default.location}-${config_1.default.projectId}.cloudfunctions.net/${instanceId}-reidentifyData' - ); - END; - `; - const options = { - query: query, - location: config_1.default.location, - }; - // Run the query as a job - const [job] = await bigqueryClient.createQueryJob(options); - functions.logger.debug(`Job ${job.id} started.`); - // Wait for the query to finish - await job.getQueryResults(); - await runtime.setProcessingState("PROCESSING_COMPLETE", "Connections created successfully."); - } - catch (error) { - if (error["code"] === 6) { - functions.logger.warn(`Functions already exists.`); - } - else { - functions.logger.error(error); - await runtime.setProcessingState("PROCESSING_FAILED", "Connections were not created, check logs for more details."); - return; - } - } -}); -//# sourceMappingURL=index.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/index.js.map b/bigquery-dlp-function/functions/lib/index.js.map deleted file mode 100644 index 27031d7a..00000000 --- a/bigquery-dlp-function/functions/lib/index.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;AAAA,gDAAgD;AAChD,wCAAwC;AACxC,2CAAoC;AACpC,2EAA4E;AAC5E,qDAAkD;AAClD,0DAA0D;AAE1D,qCAA8B;AAC9B,6CAGsB;AACtB,uDAK2B;AAC3B,6CAAqE;AAErE,KAAK,CAAC,aAAa,EAAE,CAAC;AAEtB,MAAM,GAAG,GAAG,IAAI,aAAG,CAAC,gBAAgB,EAAE,CAAC;AACvC,MAAM,cAAc,GAAG,IAAI,mBAAQ,EAAE,CAAC;AACtC,MAAM,wBAAwB,GAAG,IAAI,6CAAuB,EAAE,CAAC;AAE/D,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAChE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB,KAAK,QAAQ;YACZ,cAAc,GAAG,IAAI,sCAAoB,EAAE,CAAC;YAC5C,MAAM;QACP,KAAK,OAAO;YACX,cAAc,GAAG,IAAI,uCAAqB,EAAE,CAAC;YAC7C,MAAM;QACP,KAAK,qBAAqB;YACzB,cAAc,GAAG,IAAI,mDAAiC,EAAE,CAAC;YACzD,MAAM;QACP;YACC,cAAc,GAAG,IAAI,oCAAkB,EAAE,CAAC;KAC3C;IAED,IAAI;QACH,QAAQ,gBAAM,CAAC,MAAM,EAAE;YACtB,KAAK,WAAW;gBACf,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP,KAAK,QAAQ;gBACZ,QAAQ,CAAC,IAAI,CAAC;oBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;iBACD,CAAC,CAAC;gBACH,MAAM;YACP;gBACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;gBAC9D,MAAM;SACP;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAE9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,cAAc,GAAG,SAAS,CAAC,KAAK,CAAC,SAAS,CACjD,KAAK,EAAE,OAAO,EAAE,QAAQ,EAAE,EAAE;IAC3B,MAAM,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAE/B,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,gCAAgC,EAAE,KAAK,CAAC,CAAC;IAEhE,IAAI,cAAc,CAAC;IAEnB,QAAQ,gBAAM,CAAC,SAAS,EAAE;QACzB;YACC,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,mCAAmC,CAAC,CAAC;YAC/D,OAAO;KACR;IAED,IAAI;QACH,IAAI,gBAAM,CAAC,MAAM,KAAK,WAAW,EAAE;YAClC,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,kDAAqC,EACnD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM,IAAI,gBAAM,CAAC,MAAM,KAAK,QAAQ,EAAE;YACtC,QAAQ,CAAC,IAAI,CAAC;gBACb,OAAO,EAAE,MAAM,IAAA,gDAAmC,EACjD,KAAK,EACL,GAAG,EACH,cAAc,CACd;aACD,CAAC,CAAC;SACH;aAAM;YACN,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,gBAAgB,CAAC,CAAC;SAC5C;KACD;IAAC,OAAO,KAAK,EAAE;QACf,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;QAC9B,QAAQ,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;KACnD;AACF,CAAC,CACD,CAAC;AAEF,OAAO,CAAC,wBAAwB,GAAG,SAAS,CAAC,KAAK;KAChD,SAAS,EAAE;KACX,UAAU,CAAC,KAAK,IAAI,EAAE;IACtB,MAAM,OAAO,GAAG,IAAA,0BAAa,GAAE,CAAC,OAAO,EAAE,CAAC;IAE1C,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,MAAM,UAAU,GAAG,MAAM,GAAG,gBAAM,CAAC,aAAa,CAAC;IACjD,IAAI,UAAU,CAAC;IAEf,IAAI;QACH,UAAU,GAAG,MAAM,wBAAwB,CAAC,gBAAgB,CAAC;YAC5D,MAAM,EAAE,MAAM;YACd,YAAY,EAAE,UAAU;YACxB,UAAU,EAAE;gBACX,aAAa,EAAE;oBACd,gBAAgB,EAAE,GAAG,UAAU,IAAI,gBAAM,CAAC,SAAS,0BAA0B;iBAC7E;gBACD,IAAI,EAAE,UAAU;gBAChB,YAAY,EAAE,eAAe;aAC7B;SACD,CAAC,CAAC;QAEH,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,oCAAoC,EAAE,UAAU,CAAC,CAAC;KACxE;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CACpB,cAAc,UAAU,mDAAmD,CAC3E,CAAC;SACF;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAC9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,yDAAyD,CACzD,CAAC;YAEF,OAAO;SACP;KACD;IAED,IAAI;QACH,MAAM,KAAK,GAAG;;8BAEa,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;;8BAExE,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,SAAS;qCAC7B,gBAAM,CAAC,SAAS,IAAI,gBAAM,CAAC,QAAQ,IAAI,UAAU;;kCAEpD,gBAAM,CAAC,QAAQ,IAAI,gBAAM,CAAC,SAAS,uBAAuB,UAAU;;;UAG5F,CAAC;QAER,MAAM,OAAO,GAAG;YACf,KAAK,EAAE,KAAK;YACZ,QAAQ,EAAE,gBAAM,CAAC,QAAQ;SACzB,CAAC;QAEF,yBAAyB;QACzB,MAAM,CAAC,GAAG,CAAC,GAAG,MAAM,cAAc,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC;QAC3D,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,OAAO,GAAG,CAAC,EAAE,WAAW,CAAC,CAAC;QAEjD,+BAA+B;QAC/B,MAAM,GAAG,CAAC,eAAe,EAAE,CAAC;QAE5B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,qBAAqB,EACrB,mCAAmC,CACnC,CAAC;KACF;IAAC,OAAO,KAAU,EAAE;QACpB,IAAI,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE;YACxB,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;SACnD;aAAM;YACN,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;YAE9B,MAAM,OAAO,CAAC,kBAAkB,CAC/B,mBAAmB,EACnB,4DAA4D,CAC5D,CAAC;YAEF,OAAO;SACP;KACD;AACF,CAAC,CAAC,CAAC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js b/bigquery-dlp-function/functions/lib/reidentify.js deleted file mode 100644 index 51491a10..00000000 --- a/bigquery-dlp-function/functions/lib/reidentify.js +++ /dev/null @@ -1,32 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.reidentifyWithInfoTypeTransformations = void 0; -const functions = require("firebase-functions"); -const transofmrations_1 = require("./transofmrations"); -const config_1 = require("./config"); -async function reidentifyWithInfoTypeTransformations(rows, client, transformation) { - var _a; - const reidentifiedItems = []; - const parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - if (transformation instanceof transofmrations_1.MaskTransformation) { - functions.logger.debug("Mask Transformation is irreversable"); - throw new Error("Mask Transformation is irreversable"); - } - for (const row of rows) { - const data = row[0]; - functions.logger.debug(data); - for (const key in data) { - if (data.hasOwnProperty(key)) { - const element = data[key]; - const request = Object.assign(Object.assign({}, transformation.reidentifyConfig), { item: { value: element }, parent: parent }); - const [response] = await client.deidentifyContent(request); - data[key] = (_a = response.item) === null || _a === void 0 ? void 0 : _a.value; - } - } - functions.logger.debug(data); - reidentifiedItems.push(data); - } - return reidentifiedItems; -} -exports.reidentifyWithInfoTypeTransformations = reidentifyWithInfoTypeTransformations; -//# sourceMappingURL=reidentify.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/reidentify.js.map b/bigquery-dlp-function/functions/lib/reidentify.js.map deleted file mode 100644 index 815cbddf..00000000 --- a/bigquery-dlp-function/functions/lib/reidentify.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"reidentify.js","sourceRoot":"","sources":["../src/reidentify.ts"],"names":[],"mappings":";;;AAAA,gDAAgD;AAGhD,uDAA6E;AAC7E,qCAA8B;AAEvB,KAAK,UAAU,qCAAqC,CACzD,IAAQ,EACR,MAAwB,EACxB,cAAyD;;IAEzD,MAAM,iBAAiB,GAAG,EAAE,CAAC;IAE7B,MAAM,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAE3E,IAAI,cAAc,YAAY,oCAAkB,EAAE;QAChD,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,qCAAqC,CAAC,CAAC;QAC9D,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;KACxD;IAED,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACtB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAC3C,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAE7B,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;YACtB,IAAI,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,EAAE;gBAC5B,MAAM,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC;gBAC1B,MAAM,OAAO,mCACR,cAAc,CAAC,gBAAgB,KAClC,IAAI,EAAE,EAAE,KAAK,EAAE,OAAO,EAAE,EACxB,MAAM,EAAE,MAAM,GACf,CAAC;gBAEF,MAAM,CAAC,QAAQ,CAAC,GAAG,MAAM,MAAM,CAAC,iBAAiB,CAAC,OAAO,CAAC,CAAC;gBAC3D,IAAI,CAAC,GAAG,CAAC,GAAG,MAAA,QAAQ,CAAC,IAAI,0CAAE,KAAK,CAAC;aAClC;SACF;QAED,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC7B,iBAAiB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;KAC9B;IAED,OAAO,iBAAiB,CAAC;AAC3B,CAAC;AArCD,sFAqCC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js b/bigquery-dlp-function/functions/lib/transofmrations.js deleted file mode 100644 index b5ca74b3..00000000 --- a/bigquery-dlp-function/functions/lib/transofmrations.js +++ /dev/null @@ -1,227 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.tableToReplies = exports.rowsToTable = exports.ReplaceWithInfoTypeTransformation = exports.ReplaceTransformation = exports.RedactTransformation = exports.MaskTransformation = void 0; -const config_1 = require("./config"); -const dlp_1 = require("@google-cloud/dlp"); -class Transformation { - constructor() { - this.deidentifyConfig = {}; - this.reidentifyConfig = {}; - this.parent = `projects/${config_1.default.projectId}/locations/${config_1.default.location}`; - } -} -class MaskTransformation extends Transformation { - /** - * Replace a value by a mask character. - * - * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. - * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. - */ - constructor(mask, numberToMask) { - super(); - const maskingConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", - numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, - }, - }, - }, - ], - }, - })), (config_1.default.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask !== null && mask !== void 0 ? mask : "x", - numberToMask: numberToMask !== null && numberToMask !== void 0 ? numberToMask : 5, - }, - }, - }, - ], - }, - })); - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: maskingConfig, - }; - } -} -exports.MaskTransformation = MaskTransformation; -class RedactTransformation extends Transformation { - /** - * Redacts a value by removing it. - */ - constructor() { - super(); - const redactConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - redactConfig: {}, - }, - }, - ], - }, - })), (config_1.default.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - redactConfig: {}, - }, - }, - ], - }, - })); - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: redactConfig, - }; - } -} -exports.RedactTransformation = RedactTransformation; -class ReplaceTransformation extends Transformation { - /** - * Replace with a specified value. - */ - constructor() { - super(); - const _replaceConfig = { - newValue: { - // TODO make configurable? - stringValue: "REPLACED", - }, - }; - const replaceConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - replaceConfig: _replaceConfig, - }, - }, - ], - }, - })), (config_1.default.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - replaceConfig: _replaceConfig, - }, - }, - ], - }, - })); - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: replaceConfig, - }; - } -} -exports.ReplaceTransformation = ReplaceTransformation; -class ReplaceWithInfoTypeTransformation extends Transformation { - /** - * Replace with a specified value. - */ - constructor() { - super(); - const _replaceConfig = { - // TODO make configurable? - partToExtract: "MONTH", - }; - const replaceConfig = Object.assign(Object.assign({}, (config_1.default.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - replaceWithInfoTypeConfig: _replaceConfig, - }, - }, - ], - }, - })), (config_1.default.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - replaceWithInfoTypeConfig: _replaceConfig, - }, - }, - ], - }, - })); - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: replaceConfig, - }; - } -} -exports.ReplaceWithInfoTypeTransformation = ReplaceWithInfoTypeTransformation; -function rowsToTable(rows) { - let table = { - headers: [], - rows: [], - }; - for (const row of rows) { - const data = row[0]; - const keys = Object.keys(data); - const values = Object.values(data); - if (table.headers.length === 0) { - // Add headers only once - table.headers = keys.map((key) => { - const field = dlp_1.protos.google.privacy.dlp.v2.FieldId.create({ - name: key, - }); - return field; - }); - } - const tableRow = dlp_1.protos.google.privacy.dlp.v2.Table.Row.create({ - values: values.map((v) => { - const field = dlp_1.protos.google.privacy.dlp.v2.Value.create({ - stringValue: v, - }); - return field; - }), - }); - table.rows.push(tableRow); - } - return table; -} -exports.rowsToTable = rowsToTable; -function getFieldIds() { - var _a; - const fieldIds = (_a = config_1.default.fields) === null || _a === void 0 ? void 0 : _a.map((field) => { - return { name: field }; - }); - return fieldIds; -} -function tableToReplies(table) { - var _a; - const replies = []; - const rows = (_a = table === null || table === void 0 ? void 0 : table.rows) === null || _a === void 0 ? void 0 : _a.map((row) => { var _a; return (_a = row.values) === null || _a === void 0 ? void 0 : _a.map((value) => value.stringValue); }); - if (!rows || !table || !table.headers) - return []; - for (const row of rows) { - const reply = {}; - for (let i = 0; i < table.headers.length; i++) { - const header = table.headers[i].name; - reply[header] = row[i]; - } - replies.push(reply); - } - return replies; -} -exports.tableToReplies = tableToReplies; -//# sourceMappingURL=transofmrations.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/transofmrations.js.map b/bigquery-dlp-function/functions/lib/transofmrations.js.map deleted file mode 100644 index 8a845575..00000000 --- a/bigquery-dlp-function/functions/lib/transofmrations.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"transofmrations.js","sourceRoot":"","sources":["../src/transofmrations.ts"],"names":[],"mappings":";;;AAAA,qCAA8B;AAC9B,2CAA2C;AAQ3C,MAAM,cAAc;IAKnB;QAHA,qBAAgB,GAAsB,EAAE,CAAC;QACzC,qBAAgB,GAAsB,EAAE,CAAC;QAGxC,IAAI,CAAC,MAAM,GAAG,YAAY,gBAAM,CAAC,SAAS,cAAc,gBAAM,CAAC,QAAQ,EAAE,CAAC;IAC3E,CAAC;CACD;AAED,MAAa,kBAAmB,SAAQ,cAAc;IACrD;;;;;OAKG;IACH,YAAY,IAAa,EAAE,YAAqB;QAC/C,KAAK,EAAE,CAAC;QACR,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,mBAAmB,EAAE;gCACpB,gBAAgB,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,GAAG;gCAC7B,YAAY,EAAE,YAAY,aAAZ,YAAY,cAAZ,YAAY,GAAI,CAAC;6BAC/B;yBACD;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA9CD,gDA8CC;AAED,MAAa,oBAAqB,SAAQ,cAAc;IACvD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,YAAY,mCACd,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,YAAY,EAAE,EAAE;yBAChB;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,YAAY;SAC9B,CAAC;IACH,CAAC;CACD;AArCD,oDAqCC;AAED,MAAa,qBAAsB,SAAQ,cAAc;IACxD;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,QAAQ,EAAE;gBACT,0BAA0B;gBAC1B,WAAW,EAAE,UAAU;aACvB;SACD,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,aAAa,EAAE,cAAc;yBAC7B;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA5CD,sDA4CC;AAED,MAAa,iCAAkC,SAAQ,cAAc;IACpE;;OAEG;IACH;QACC,KAAK,EAAE,CAAC;QACR,MAAM,cAAc,GAAG;YACtB,0BAA0B;YAC1B,aAAa,EAAE,OAAO;SACtB,CAAC;QAEF,MAAM,aAAa,mCACf,CAAC,gBAAM,CAAC,MAAM,IAAI,WAAW,IAAI;YACnC,uBAAuB,EAAE;gBACxB,eAAe,EAAE;oBAChB;wBACC,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,GACC,CAAC,gBAAM,CAAC,MAAM,IAAI,QAAQ,IAAI;YAChC,qBAAqB,EAAE;gBACtB,oBAAoB,EAAE;oBACrB;wBACC,MAAM,EAAE,WAAW,EAAE;wBACrB,uBAAuB,EAAE;4BACxB,yBAAyB,EAAE,cAAc;yBACzC;qBACD;iBACD;aACD;SACD,CAAC,CACF,CAAC;QAEF,IAAI,CAAC,gBAAgB,GAAG;YACvB,MAAM,EAAE,IAAI,CAAC,MAAM;YACnB,gBAAgB,EAAE,aAAa;SAC/B,CAAC;IACH,CAAC;CACD;AA1CD,8EA0CC;AAED,SAAgB,WAAW,CAAC,IAAQ;IACnC,IAAI,KAAK,GAAG;QACX,OAAO,EAAE,EAAe;QACxB,IAAI,EAAE,EAAW;KACjB,CAAC;IAEF,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,IAAI,GAAG,GAAG,CAAC,CAAC,CAAwB,CAAC;QAE3C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC/B,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QAEnC,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE;YAC/B,wBAAwB;YACxB,KAAK,CAAC,OAAO,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE;gBAChC,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC;oBACzD,IAAI,EAAE,GAAG;iBACT,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;SACH;QAED,MAAM,QAAQ,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,CAAC;YAC9D,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;gBACxB,MAAM,KAAK,GAAG,YAAM,CAAC,MAAM,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC,KAAK,CAAC,MAAM,CAAC;oBACvD,WAAW,EAAE,CAAC;iBACd,CAAC,CAAC;gBACH,OAAO,KAAK,CAAC;YACd,CAAC,CAAC;SACF,CAAC,CAAC;QAEH,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;KAC1B;IAED,OAAO,KAAK,CAAC;AACd,CAAC;AAnCD,kCAmCC;AAED,SAAS,WAAW;;IACnB,MAAM,QAAQ,GAAG,MAAA,gBAAM,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE;QAC7C,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC;IACxB,CAAC,CAAC,CAAC;IACH,OAAO,QAAQ,CAAC;AACjB,CAAC;AAED,SAAgB,cAAc,CAAC,KAAY;;IAC1C,MAAM,OAAO,GAAG,EAAE,CAAC;IACnB,MAAM,IAAI,GAAG,MAAA,KAAK,aAAL,KAAK,uBAAL,KAAK,CAAE,IAAI,0CAAE,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,WACrC,OAAA,MAAA,GAAG,CAAC,MAAM,0CAAE,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,WAAW,CAAC,CAAA,EAAA,CAC7C,CAAC;IAEF,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,CAAC,KAAK,CAAC,OAAO;QAAE,OAAO,EAAE,CAAC;IAEjD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE;QACvB,MAAM,KAAK,GAAG,EAAyB,CAAC;QAExC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE;YAC9C,MAAM,MAAM,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAc,CAAC;YAC/C,KAAK,CAAC,MAAM,CAAC,GAAG,GAAI,CAAC,CAAC,CAAC,CAAC;SACxB;QAED,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;KACpB;IAED,OAAO,OAAO,CAAC;AAChB,CAAC;AApBD,wCAoBC"} \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/types/call_mode.js b/bigquery-dlp-function/functions/lib/types/call_mode.js deleted file mode 100644 index 2a20c9ac..00000000 --- a/bigquery-dlp-function/functions/lib/types/call_mode.js +++ /dev/null @@ -1,9 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -var CallMode; -(function (CallMode) { - CallMode["DEIDENTIFY"] = "DEIDENTIFY"; - CallMode["REIDENTIFY"] = "REIDENTIFY"; -})(CallMode || (CallMode = {})); -exports.default = CallMode; -//# sourceMappingURL=call_mode.js.map \ No newline at end of file diff --git a/bigquery-dlp-function/functions/lib/types/call_mode.js.map b/bigquery-dlp-function/functions/lib/types/call_mode.js.map deleted file mode 100644 index 3f8263e5..00000000 --- a/bigquery-dlp-function/functions/lib/types/call_mode.js.map +++ /dev/null @@ -1 +0,0 @@ -{"version":3,"file":"call_mode.js","sourceRoot":"","sources":["../../src/types/call_mode.ts"],"names":[],"mappings":";;AAAA,IAAK,QAGJ;AAHD,WAAK,QAAQ;IACX,qCAAyB,CAAA;IACzB,qCAAyB,CAAA;AAC3B,CAAC,EAHI,QAAQ,KAAR,QAAQ,QAGZ;AAED,kBAAe,QAAQ,CAAC"} \ No newline at end of file From 7c741cd624fdfd029c6d915aa09ef081bb736f14 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Fri, 7 Apr 2023 17:54:02 +0100 Subject: [PATCH 34/37] feat: update documentation --- bigquery-dlp-function/extension.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-function/extension.yaml index e241cda1..c4ed696e 100644 --- a/bigquery-dlp-function/extension.yaml +++ b/bigquery-dlp-function/extension.yaml @@ -11,7 +11,7 @@ author: displayName: BigQuery DLP Remote Function -description: TODO +description: This extension creates BigQuery functions to facilitate de-identification and re-identification in queries, providing configurable techniques, seamless integration, and ensuring better data privacy and compliance. license: Apache-2.0 # https://spdx.org/licenses/ From 8b7ab2e9a0f45375b2670e1cf88b25d85f82c1f3 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Fri, 7 Apr 2023 18:28:38 +0100 Subject: [PATCH 35/37] chore: ran formatting --- bigquery-dlp-function/functions/src/config.ts | 14 +- .../functions/src/deidentify.ts | 112 ++--- bigquery-dlp-function/functions/src/index.ts | 330 +++++++------- .../functions/src/transofmrations.ts | 422 +++++++++--------- bigquery-dlp-function/functions/tsconfig.json | 4 +- 5 files changed, 440 insertions(+), 442 deletions(-) diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-function/functions/src/config.ts index 4b84d99f..480a2e5e 100644 --- a/bigquery-dlp-function/functions/src/config.ts +++ b/bigquery-dlp-function/functions/src/config.ts @@ -15,11 +15,11 @@ */ export default { - projectId: process.env.PROJECT_ID, - extInstanceId: process.env.EXT_INSTANCE_ID, - datasetId: process.env.DATASET_ID, - location: process.env.LOCATION, - method: process.env.TRANSFORMATION_METHOD, - technique: process.env.TRANSFORMATION_TECHNIQUE, - fields: process.env.FIELDS_TO_TRANSFORM?.split(","), + projectId: process.env.PROJECT_ID, + extInstanceId: process.env.EXT_INSTANCE_ID, + datasetId: process.env.DATASET_ID, + location: process.env.LOCATION, + method: process.env.TRANSFORMATION_METHOD, + technique: process.env.TRANSFORMATION_TECHNIQUE, + fields: process.env.FIELDS_TO_TRANSFORM?.split(","), }; diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-function/functions/src/deidentify.ts index e72a6ffd..01233c87 100644 --- a/bigquery-dlp-function/functions/src/deidentify.ts +++ b/bigquery-dlp-function/functions/src/deidentify.ts @@ -2,12 +2,12 @@ import * as functions from "firebase-functions"; import { DlpServiceClient } from "@google-cloud/dlp"; import { - MaskTransformation, - RedactTransformation, - ReplaceTransformation, - ReplaceWithInfoTypeTransformation, - rowsToTable, - tableToReplies, + MaskTransformation, + RedactTransformation, + ReplaceTransformation, + ReplaceWithInfoTypeTransformation, + rowsToTable, + tableToReplies, } from "./transofmrations"; /** @@ -21,36 +21,36 @@ import { * @returns {Promise>} The deidentified record. */ export async function deidentifyWithInfoTypeTransformations( - rows: [], - client: DlpServiceClient, - transformation: - | MaskTransformation - | RedactTransformation - | ReplaceTransformation - | ReplaceWithInfoTypeTransformation + rows: [], + client: DlpServiceClient, + transformation: + | MaskTransformation + | RedactTransformation + | ReplaceTransformation + | ReplaceWithInfoTypeTransformation ): Promise> { - const deidentifiedItems = []; + const deidentifiedItems = []; - for (const row of rows) { - const data = row[0] as Record; + for (const row of rows) { + const data = row[0] as Record; - for (const key in data) { - if (data.hasOwnProperty(key)) { - const element = data[key]; - const request = { - ...transformation.deidentifyConfig, - item: { value: element }, - }; + for (const key in data) { + if (data.hasOwnProperty(key)) { + const element = data[key]; + const request = { + ...transformation.deidentifyConfig, + item: { value: element }, + }; - const [response] = await client.deidentifyContent(request); - data[key] = response.item?.value; - } - } + const [response] = await client.deidentifyContent(request); + data[key] = response.item?.value; + } + } - deidentifiedItems.push(data); - } + deidentifiedItems.push(data); + } - return deidentifiedItems; + return deidentifiedItems; } /** @@ -64,36 +64,36 @@ export async function deidentifyWithInfoTypeTransformations( * @returns {Promise} The deidentified text. */ export async function deidentifyWithRecordTransformations( - rows: [], - client: DlpServiceClient, - transformation: - | MaskTransformation - | RedactTransformation - | ReplaceTransformation - | ReplaceWithInfoTypeTransformation + rows: [], + client: DlpServiceClient, + transformation: + | MaskTransformation + | RedactTransformation + | ReplaceTransformation + | ReplaceWithInfoTypeTransformation ) { - let table; + let table; - try { - // Convert raw rows to Table type - table = rowsToTable(rows); - } catch (error) { - functions.logger.debug(`Error converting rows to Table type.`); - throw error; - } + try { + // Convert raw rows to Table type + table = rowsToTable(rows); + } catch (error) { + functions.logger.debug(`Error converting rows to Table type.`); + throw error; + } - // Construct de-identification request - const request = { - ...transformation.deidentifyConfig, - item: { - table: table, - }, - }; + // Construct de-identification request + const request = { + ...transformation.deidentifyConfig, + item: { + table: table, + }, + }; - // Run deidentification request - const [response] = await client.deidentifyContent(request); + // Run deidentification request + const [response] = await client.deidentifyContent(request); - functions.logger.debug(tableToReplies(response.item?.table)); + functions.logger.debug(tableToReplies(response.item?.table)); - return tableToReplies(response.item?.table); + return tableToReplies(response.item?.table); } diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-function/functions/src/index.ts index 4d9c0333..098bc582 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-function/functions/src/index.ts @@ -7,14 +7,14 @@ import { getExtensions } from "firebase-admin/extensions"; import config from "./config"; import { - deidentifyWithInfoTypeTransformations, - deidentifyWithRecordTransformations, + deidentifyWithInfoTypeTransformations, + deidentifyWithRecordTransformations, } from "./deidentify"; import { - MaskTransformation, - RedactTransformation, - ReplaceTransformation, - ReplaceWithInfoTypeTransformation, + MaskTransformation, + RedactTransformation, + ReplaceTransformation, + ReplaceWithInfoTypeTransformation, } from "./transofmrations"; import { reidentifyWithInfoTypeTransformations } from "./reidentify"; @@ -25,140 +25,140 @@ const bigqueryClient = new BigQuery(); const bigqueryConnectionClient = new ConnectionServiceClient(); exports.deidentifyData = functions.https.onRequest( - async (request, response) => { - const { calls } = request.body; - - functions.logger.debug("Incoming request from BigQuery", calls); - var transformation; - - switch (config.technique) { - case "redact": - transformation = new RedactTransformation(); - break; - case "fixed": - transformation = new ReplaceTransformation(); - break; - case "replaceWithInfoType": - transformation = new ReplaceWithInfoTypeTransformation(); - break; - default: - transformation = new MaskTransformation(); - } - - try { - switch (config.method) { - case "INFO_TYPE": - response.send({ - replies: await deidentifyWithInfoTypeTransformations( - calls, - dlp, - transformation - ), - }); - break; - case "RECORD": - response.send({ - replies: await deidentifyWithRecordTransformations( - calls, - dlp, - transformation - ), - }); - break; - default: - response.status(400).send({ errorMessage: "Invalid method" }); - break; - } - } catch (error) { - functions.logger.error(error); - - response.status(400).send({ errorMessage: error }); - } - } + async (request, response) => { + const { calls } = request.body; + + functions.logger.debug("Incoming request from BigQuery", calls); + var transformation; + + switch (config.technique) { + case "redact": + transformation = new RedactTransformation(); + break; + case "fixed": + transformation = new ReplaceTransformation(); + break; + case "replaceWithInfoType": + transformation = new ReplaceWithInfoTypeTransformation(); + break; + default: + transformation = new MaskTransformation(); + } + + try { + switch (config.method) { + case "INFO_TYPE": + response.send({ + replies: await deidentifyWithInfoTypeTransformations( + calls, + dlp, + transformation + ), + }); + break; + case "RECORD": + response.send({ + replies: await deidentifyWithRecordTransformations( + calls, + dlp, + transformation + ), + }); + break; + default: + response.status(400).send({ errorMessage: "Invalid method" }); + break; + } + } catch (error) { + functions.logger.error(error); + + response.status(400).send({ errorMessage: error }); + } + } ); exports.reidentifyData = functions.https.onRequest( - async (request, response) => { - const { calls } = request.body; - - functions.logger.debug("Incoming request from BigQuery", calls); - - var transformation; - - switch (config.technique) { - default: - response.status(400).send("Invalid or irreversable technique"); - return; - } - - try { - if (config.method === "INFO_TYPE") { - response.send({ - replies: await reidentifyWithInfoTypeTransformations( - calls, - dlp, - transformation - ), - }); - } else if (config.method === "RECORD") { - response.send({ - replies: await deidentifyWithRecordTransformations( - calls, - dlp, - transformation - ), - }); - } else { - response.status(400).send("Invalid method"); - } - } catch (error) { - functions.logger.error(error); - response.status(400).send({ errorMessage: error }); - } - } + async (request, response) => { + const { calls } = request.body; + + functions.logger.debug("Incoming request from BigQuery", calls); + + var transformation; + + switch (config.technique) { + default: + response.status(400).send("Invalid or irreversable technique"); + return; + } + + try { + if (config.method === "INFO_TYPE") { + response.send({ + replies: await reidentifyWithInfoTypeTransformations( + calls, + dlp, + transformation + ), + }); + } else if (config.method === "RECORD") { + response.send({ + replies: await deidentifyWithRecordTransformations( + calls, + dlp, + transformation + ), + }); + } else { + response.status(400).send("Invalid method"); + } + } catch (error) { + functions.logger.error(error); + response.status(400).send({ errorMessage: error }); + } + } ); exports.createBigQueryConnection = functions.tasks - .taskQueue() - .onDispatch(async () => { - const runtime = getExtensions().runtime(); - - const parent = `projects/${config.projectId}/locations/${config.location}`; - const instanceId = "ext-" + config.extInstanceId; - var connection; - - try { - connection = await bigqueryConnectionClient.createConnection({ - parent: parent, - connectionId: instanceId, - connection: { - cloudResource: { - serviceAccountId: `${instanceId}@${config.projectId}.iam.gserviceaccount.com`, - }, - name: instanceId, - friendlyName: "DLP Extension", - }, - }); - - functions.logger.info("Connection successfully created 🎉", connection); - } catch (error: any) { - if (error["code"] === 6) { - functions.logger.warn( - `Connection ${instanceId} already exists, will continue creating functions` - ); - } else { - functions.logger.error(error); - await runtime.setProcessingState( - "PROCESSING_FAILED", - "Error creating connection. Check logs for more details." - ); - - return; - } - } - - try { - const query = ` + .taskQueue() + .onDispatch(async () => { + const runtime = getExtensions().runtime(); + + const parent = `projects/${config.projectId}/locations/${config.location}`; + const instanceId = "ext-" + config.extInstanceId; + var connection; + + try { + connection = await bigqueryConnectionClient.createConnection({ + parent: parent, + connectionId: instanceId, + connection: { + cloudResource: { + serviceAccountId: `${instanceId}@${config.projectId}.iam.gserviceaccount.com`, + }, + name: instanceId, + friendlyName: "DLP Extension", + }, + }); + + functions.logger.info("Connection successfully created 🎉", connection); + } catch (error: any) { + if (error["code"] === 6) { + functions.logger.warn( + `Connection ${instanceId} already exists, will continue creating functions` + ); + } else { + functions.logger.error(error); + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Error creating connection. Check logs for more details." + ); + + return; + } + } + + try { + const query = ` BEGIN CREATE FUNCTION \`${config.projectId}.${config.datasetId}\`.deidentify(data JSON) RETURNS JSON REMOTE WITH CONNECTION \`${config.projectId}.${config.location}.${instanceId}\` @@ -173,34 +173,34 @@ exports.createBigQueryConnection = functions.tasks END; `; - const options = { - query: query, - location: config.location, - }; - - // Run the query as a job - const [job] = await bigqueryClient.createQueryJob(options); - functions.logger.debug(`Job ${job.id} started.`); - - // Wait for the query to finish - await job.getQueryResults(); - - await runtime.setProcessingState( - "PROCESSING_COMPLETE", - "Connections created successfully." - ); - } catch (error: any) { - if (error["code"] === 6) { - functions.logger.warn(`Functions already exists.`); - } else { - functions.logger.error(error); - - await runtime.setProcessingState( - "PROCESSING_FAILED", - "Connections were not created, check logs for more details." - ); - - return; - } - } - }); + const options = { + query: query, + location: config.location, + }; + + // Run the query as a job + const [job] = await bigqueryClient.createQueryJob(options); + functions.logger.debug(`Job ${job.id} started.`); + + // Wait for the query to finish + await job.getQueryResults(); + + await runtime.setProcessingState( + "PROCESSING_COMPLETE", + "Connections created successfully." + ); + } catch (error: any) { + if (error["code"] === 6) { + functions.logger.warn(`Functions already exists.`); + } else { + functions.logger.error(error); + + await runtime.setProcessingState( + "PROCESSING_FAILED", + "Connections were not created, check logs for more details." + ); + + return; + } + } + }); diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-function/functions/src/transofmrations.ts index 49e4ae2a..223f588c 100644 --- a/bigquery-dlp-function/functions/src/transofmrations.ts +++ b/bigquery-dlp-function/functions/src/transofmrations.ts @@ -8,254 +8,254 @@ type Row = protos.google.privacy.dlp.v2.Table.Row; type Table = protos.google.privacy.dlp.v2.ITable | undefined | null; class Transformation { - parent: string; - deidentifyConfig: DeidentifyRequest = {}; - reidentifyConfig: ReidentifyRequest = {}; + parent: string; + deidentifyConfig: DeidentifyRequest = {}; + reidentifyConfig: ReidentifyRequest = {}; - constructor() { - this.parent = `projects/${config.projectId}/locations/${config.location}`; - } + constructor() { + this.parent = `projects/${config.projectId}/locations/${config.location}`; + } } export class MaskTransformation extends Transformation { - /** - * Replace a value by a mask character. - * - * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. - * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. - */ - constructor(mask?: string, numberToMask?: number) { - super(); - const maskingConfig = { - ...(config.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, - }, - }, - }, - ], - }, - }), - ...(config.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - characterMaskConfig: { - maskingCharacter: mask ?? "x", - numberToMask: numberToMask ?? 5, - }, - }, - }, - ], - }, - }), - }; + /** + * Replace a value by a mask character. + * + * @param mask The character to mask the sensitive data with. If not supplied, defaults to `x`. + * @param numberToMask The number of characters to mask. If not supplied, defaults to `5`. + */ + constructor(mask?: string, numberToMask?: number) { + super(); + const maskingConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + characterMaskConfig: { + maskingCharacter: mask ?? "x", + numberToMask: numberToMask ?? 5, + }, + }, + }, + ], + }, + }), + }; - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: maskingConfig, - }; - } + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: maskingConfig, + }; + } } export class RedactTransformation extends Transformation { - /** - * Redacts a value by removing it. - */ - constructor() { - super(); - const redactConfig = { - ...(config.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - redactConfig: {}, - }, - }, - ], - }, - }), - ...(config.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - redactConfig: {}, - }, - }, - ], - }, - }), - }; + /** + * Redacts a value by removing it. + */ + constructor() { + super(); + const redactConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + redactConfig: {}, + }, + }, + ], + }, + }), + }; - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: redactConfig, - }; - } + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: redactConfig, + }; + } } export class ReplaceTransformation extends Transformation { - /** - * Replace with a specified value. - */ - constructor() { - super(); - const _replaceConfig = { - newValue: { - // TODO make configurable? - stringValue: "REPLACED", - }, - }; + /** + * Replace with a specified value. + */ + constructor() { + super(); + const _replaceConfig = { + newValue: { + // TODO make configurable? + stringValue: "REPLACED", + }, + }; - const replaceConfig = { - ...(config.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - replaceConfig: _replaceConfig, - }, - }, - ], - }, - }), - ...(config.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - replaceConfig: _replaceConfig, - }, - }, - ], - }, - }), - }; + const replaceConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + replaceConfig: _replaceConfig, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + replaceConfig: _replaceConfig, + }, + }, + ], + }, + }), + }; - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: replaceConfig, - }; - } + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: replaceConfig, + }; + } } export class ReplaceWithInfoTypeTransformation extends Transformation { - /** - * Replace with a specified value. - */ - constructor() { - super(); - const _replaceConfig = { - // TODO make configurable? - partToExtract: "MONTH", - }; + /** + * Replace with a specified value. + */ + constructor() { + super(); + const _replaceConfig = { + // TODO make configurable? + partToExtract: "MONTH", + }; - const replaceConfig = { - ...(config.method == "INFO_TYPE" && { - infoTypeTransformations: { - transformations: [ - { - primitiveTransformation: { - replaceWithInfoTypeConfig: _replaceConfig, - }, - }, - ], - }, - }), - ...(config.method == "RECORD" && { - recordTransformations: { - fieldTransformations: [ - { - fields: getFieldIds(), - primitiveTransformation: { - replaceWithInfoTypeConfig: _replaceConfig, - }, - }, - ], - }, - }), - }; + const replaceConfig = { + ...(config.method == "INFO_TYPE" && { + infoTypeTransformations: { + transformations: [ + { + primitiveTransformation: { + replaceWithInfoTypeConfig: _replaceConfig, + }, + }, + ], + }, + }), + ...(config.method == "RECORD" && { + recordTransformations: { + fieldTransformations: [ + { + fields: getFieldIds(), + primitiveTransformation: { + replaceWithInfoTypeConfig: _replaceConfig, + }, + }, + ], + }, + }), + }; - this.deidentifyConfig = { - parent: this.parent, - deidentifyConfig: replaceConfig, - }; - } + this.deidentifyConfig = { + parent: this.parent, + deidentifyConfig: replaceConfig, + }; + } } export function rowsToTable(rows: []) { - let table = { - headers: [] as FieldId[], - rows: [] as Row[], - }; + let table = { + headers: [] as FieldId[], + rows: [] as Row[], + }; - for (const row of rows) { - const data = row[0] as Record; + for (const row of rows) { + const data = row[0] as Record; - const keys = Object.keys(data); - const values = Object.values(data); + const keys = Object.keys(data); + const values = Object.values(data); - if (table.headers.length === 0) { - // Add headers only once - table.headers = keys.map((key) => { - const field = protos.google.privacy.dlp.v2.FieldId.create({ - name: key, - }); - return field; - }); - } + if (table.headers.length === 0) { + // Add headers only once + table.headers = keys.map((key) => { + const field = protos.google.privacy.dlp.v2.FieldId.create({ + name: key, + }); + return field; + }); + } - const tableRow = protos.google.privacy.dlp.v2.Table.Row.create({ - values: values.map((v) => { - const field = protos.google.privacy.dlp.v2.Value.create({ - stringValue: v, - }); - return field; - }), - }); + const tableRow = protos.google.privacy.dlp.v2.Table.Row.create({ + values: values.map((v) => { + const field = protos.google.privacy.dlp.v2.Value.create({ + stringValue: v, + }); + return field; + }), + }); - table.rows.push(tableRow); - } + table.rows.push(tableRow); + } - return table; + return table; } function getFieldIds() { - const fieldIds = config.fields?.map((field) => { - return { name: field }; - }); - return fieldIds; + const fieldIds = config.fields?.map((field) => { + return { name: field }; + }); + return fieldIds; } export function tableToReplies(table: Table) { - const replies = []; - const rows = table?.rows?.map((row) => - row.values?.map((value) => value.stringValue) - ); + const replies = []; + const rows = table?.rows?.map((row) => + row.values?.map((value) => value.stringValue) + ); - if (!rows || !table || !table.headers) return []; + if (!rows || !table || !table.headers) return []; - for (const row of rows) { - const reply = {} as Record; + for (const row of rows) { + const reply = {} as Record; - for (let i = 0; i < table.headers.length; i++) { - const header = table.headers[i].name as string; - reply[header] = row![i]; - } + for (let i = 0; i < table.headers.length; i++) { + const header = table.headers[i].name as string; + reply[header] = row![i]; + } - replies.push(reply); - } + replies.push(reply); + } - return replies; + return replies; } diff --git a/bigquery-dlp-function/functions/tsconfig.json b/bigquery-dlp-function/functions/tsconfig.json index 7ce05d03..a9ed863a 100644 --- a/bigquery-dlp-function/functions/tsconfig.json +++ b/bigquery-dlp-function/functions/tsconfig.json @@ -9,7 +9,5 @@ "target": "es2017" }, "compileOnSave": true, - "include": [ - "src" - ] + "include": ["src"] } From 4677ce7a2191ac7684a953ecc26e831dd211fe6f Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Thu, 13 Apr 2023 13:37:47 +0100 Subject: [PATCH 36/37] frat: renamed folder and added jest testing --- .../bigquery-dlp-functions.env.local | 0 _emulator/firebase.json | 2 - .../CHANGELOG.md | 0 .../POSTINSTALL.md | 0 .../PREINSTALL.md | 0 .../README.md | 0 .../extension.yaml | 0 .../functions/.gitignore | 0 .../functions/__tests__/__mocks__/index.ts | 49 ++ .../functions/__tests__/functions.test.ts | 442 ++++++++++++++++++ .../functions/__tests__/jest.setup.ts | 15 + .../functions/__tests__/setupEnvironment.ts | 8 + .../functions/__tests__/tsconfig.json | 4 + .../functions/package.json | 9 +- .../functions/src/config.ts | 0 .../functions/src/deidentify.ts | 0 .../functions/src/index.ts | 7 +- .../functions/src/reidentify.ts | 0 .../functions/src/transofmrations.ts | 0 .../functions/tsconfig.json | 0 20 files changed, 528 insertions(+), 8 deletions(-) create mode 100644 _emulator/extensions/bigquery-dlp-functions.env.local rename {bigquery-dlp-function => bigquery-dlp-functions}/CHANGELOG.md (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/POSTINSTALL.md (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/PREINSTALL.md (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/README.md (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/extension.yaml (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/.gitignore (100%) create mode 100644 bigquery-dlp-functions/functions/__tests__/__mocks__/index.ts create mode 100644 bigquery-dlp-functions/functions/__tests__/functions.test.ts create mode 100644 bigquery-dlp-functions/functions/__tests__/jest.setup.ts create mode 100644 bigquery-dlp-functions/functions/__tests__/setupEnvironment.ts create mode 100644 bigquery-dlp-functions/functions/__tests__/tsconfig.json rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/package.json (83%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/src/config.ts (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/src/deidentify.ts (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/src/index.ts (97%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/src/reidentify.ts (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/src/transofmrations.ts (100%) rename {bigquery-dlp-function => bigquery-dlp-functions}/functions/tsconfig.json (100%) diff --git a/_emulator/extensions/bigquery-dlp-functions.env.local b/_emulator/extensions/bigquery-dlp-functions.env.local new file mode 100644 index 00000000..e69de29b diff --git a/_emulator/firebase.json b/_emulator/firebase.json index 597bab84..89804c34 100644 --- a/_emulator/firebase.json +++ b/_emulator/firebase.json @@ -1,7 +1,5 @@ { "extensions": { - "firestore-record-user-acknowledgements": "../firestore-record-user-acknowledgements", - "firestore-bundle-server": "../firestore-bundle-server", "bigquery-dlp-functions": "../bigquery-dlp-functions" }, "storage": { diff --git a/bigquery-dlp-function/CHANGELOG.md b/bigquery-dlp-functions/CHANGELOG.md similarity index 100% rename from bigquery-dlp-function/CHANGELOG.md rename to bigquery-dlp-functions/CHANGELOG.md diff --git a/bigquery-dlp-function/POSTINSTALL.md b/bigquery-dlp-functions/POSTINSTALL.md similarity index 100% rename from bigquery-dlp-function/POSTINSTALL.md rename to bigquery-dlp-functions/POSTINSTALL.md diff --git a/bigquery-dlp-function/PREINSTALL.md b/bigquery-dlp-functions/PREINSTALL.md similarity index 100% rename from bigquery-dlp-function/PREINSTALL.md rename to bigquery-dlp-functions/PREINSTALL.md diff --git a/bigquery-dlp-function/README.md b/bigquery-dlp-functions/README.md similarity index 100% rename from bigquery-dlp-function/README.md rename to bigquery-dlp-functions/README.md diff --git a/bigquery-dlp-function/extension.yaml b/bigquery-dlp-functions/extension.yaml similarity index 100% rename from bigquery-dlp-function/extension.yaml rename to bigquery-dlp-functions/extension.yaml diff --git a/bigquery-dlp-function/functions/.gitignore b/bigquery-dlp-functions/functions/.gitignore similarity index 100% rename from bigquery-dlp-function/functions/.gitignore rename to bigquery-dlp-functions/functions/.gitignore diff --git a/bigquery-dlp-functions/functions/__tests__/__mocks__/index.ts b/bigquery-dlp-functions/functions/__tests__/__mocks__/index.ts new file mode 100644 index 00000000..46f94d77 --- /dev/null +++ b/bigquery-dlp-functions/functions/__tests__/__mocks__/index.ts @@ -0,0 +1,49 @@ +export function createMockDeidentifyContentResponse(itemType: any, value: any) { + return [ + { + item: { + [itemType]: value, + }, + }, + ]; +} + +export const mockProtos = { + google: { + privacy: { + dlp: { + v2: { + FieldId: { + create: jest.fn().mockImplementation(() => { + return { test: "data" }; + }), + }, + Table: { + Row: { + create: jest.fn().mockImplementation(() => { + return { test: "data" }; + }), + }, + }, + Value: { + create: jest.fn().mockImplementation(() => { + return { test: "data" }; + }), + }, + }, + }, + }, + }, +}; + +export const getFunctions = () => { + return { + taskQueue: (functionName: any, queueName: any) => { + return { + enqueue: async (payload: any) => { + console.log("Enqueue payload:", payload); + }, + }; + }, + }; +}; diff --git a/bigquery-dlp-functions/functions/__tests__/functions.test.ts b/bigquery-dlp-functions/functions/__tests__/functions.test.ts new file mode 100644 index 00000000..d8598e0c --- /dev/null +++ b/bigquery-dlp-functions/functions/__tests__/functions.test.ts @@ -0,0 +1,442 @@ +/** Set dynamic mocking modules */ +const mockDeidentityContent = jest.fn(); +const mockCreateConnection = jest.fn(); +const mockCreateQueryJob = jest.fn(); +const mockSetProcessingState = jest.fn(); + +import { + createMockDeidentifyContentResponse, + getFunctions, + mockProtos, +} from "./__mocks__"; + +const fft = require("firebase-functions-test"); +import { DlpServiceClient } from "@google-cloud/dlp"; +import mockedConfig from "../src/config"; +import { logger } from "firebase-functions"; +import * as functions from "../src/index"; +import setupEnvironment from "./setupEnvironment"; + +const testEnv = fft({ projectId: "demo-test" }); + +/** Mock imported modules */ +jest.mock("../src/config"); + +jest.mock("firebase-admin/functions", () => { + return { getFunctions }; +}); + +jest.mock("firebase-admin/extensions", () => { + return { + getExtensions: jest.fn().mockImplementation(() => ({ + runtime: jest.fn().mockImplementation(() => ({ + setProcessingState: mockSetProcessingState, + })), + })), + }; +}); + +jest.mock("@google-cloud/bigquery", () => { + return { + BigQuery: jest.fn().mockImplementation(() => ({ + createQueryJob: mockCreateQueryJob, + })), + }; +}); + +jest.mock("@google-cloud/bigquery-connection", () => { + return { + ConnectionServiceClient: jest.fn().mockImplementation(() => ({ + createConnection: mockCreateConnection, + })), + }; +}); + +jest.mock("@google-cloud/dlp", () => { + return { + DlpServiceClient: jest.fn().mockImplementation(() => ({ + deidentifyContent: mockDeidentityContent, + })), + protos: { ...mockProtos }, + }; +}); + +/** Mock logging functions */ +const debugSpy = jest.spyOn(logger, "debug"); +const warnSpy = jest.spyOn(logger, "warn"); +const infoSpy = jest.spyOn(logger, "info"); + +setupEnvironment(); + +beforeEach(() => { + jest.resetModules(); +}); + +afterEach(() => { + jest.clearAllMocks(); +}); + +describe("BigQuery DLP Extension", () => { + const dlpClient = new DlpServiceClient(); + + describe("deidentifyData", () => { + it("should deidentify data with valid input and a redact technique and INFO_TYPE method", (done) => { + /* Set config data */ + mockedConfig.technique = "redact"; + mockedConfig.method = "INFO_TYPE"; + + /** Configure mock responses */ + mockDeidentityContent.mockImplementation(async () => { + return [ + { + item: { + value: "mock-deidentified-content", + }, + }, + ]; + }); + + /** Setup the http request and expectations */ + const req = { body: { calls: [[{ row1: [{ foo: "bar" }] }]] } }; + const res = { + send: (data: Record) => { + expect(data.replies).toEqual([{ row1: "mock-deidentified-content" }]); + done(); + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(dlpClient.deidentifyContent).toHaveBeenCalled(); + expect(debugSpy).toBeCalledWith( + "Incoming request from BigQuery", + req.body.calls + ); + }); + + it("should deidentify data with valid input and a redact technique and RECORD method", (done) => { + /* Set config data */ + mockedConfig.technique = "redact"; + mockedConfig.method = "RECORD"; + + /** Configure mock responses */ + mockDeidentityContent.mockImplementation(async () => + createMockDeidentifyContentResponse("table", { + headers: [{ name: "test_header" }], + rows: [{ values: [{ stringValue: "testing" }] }], + }) + ); + + /** Setup the http request and expectations */ + const req = { body: { calls: [[{ row1: [{ foo: "bar" }] }]] } }; + const res = { + send: (data: Record) => { + expect(data.replies).toEqual([{ test_header: "testing" }]); + done(); + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(dlpClient.deidentifyContent).toHaveBeenCalled(); + expect(debugSpy).toBeCalledWith( + "Incoming request from BigQuery", + req.body.calls + ); + }); + + it("should deidentify data with valid input and a fixed technique and INFO_TYPE method", (done) => { + /* Set config data */ + mockedConfig.technique = "fixed"; + mockedConfig.method = "INFO_TYPE"; + + /** Configure mock responses */ + mockDeidentityContent.mockImplementation(async () => + createMockDeidentifyContentResponse( + "value", + "mock-deidentified-content" + ) + ); + + /** Setup the http request and expectations */ + const req = { body: { calls: [[{ row1: [{ foo: "bar" }] }]] } }; + const res = { + send: (data: Record) => { + expect(data.replies).toEqual([{ row1: "mock-deidentified-content" }]); + done(); + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(dlpClient.deidentifyContent).toHaveBeenCalled(); + expect(debugSpy).toBeCalledWith( + "Incoming request from BigQuery", + req.body.calls + ); + }); + + it("should deidentify data with valid input and a fixed technique and RECORD method", (done) => { + /* Set config data */ + mockedConfig.technique = "fixed"; + mockedConfig.method = "RECORD"; + + /** Configure mock responses */ + mockDeidentityContent.mockImplementation(async () => + createMockDeidentifyContentResponse("table", { + headers: [{ name: "test_header" }], + rows: [{ values: [{ stringValue: "testing" }] }], + }) + ); + + /** Setup the http request and expectations */ + const req = { body: { calls: [[{ row1: [{ foo: "bar" }] }]] } }; + const res = { + send: (data: Record) => { + console.log(data); + expect(data.replies).toEqual([{ test_header: "testing" }]); + done(); + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(dlpClient.deidentifyContent).toHaveBeenCalled(); + expect(debugSpy).toBeCalledWith( + "Incoming request from BigQuery", + req.body.calls + ); + }); + + it("should deidentify data with valid input and a replaceWithInfoType technique and INFO_TYPE method", (done) => { + /* Set config data */ + mockedConfig.technique = "replaceWithInfoType"; + mockedConfig.method = "INFO_TYPE"; + + /** Configure mock responses */ + mockDeidentityContent.mockImplementation(async () => + createMockDeidentifyContentResponse( + "value", + "mock-deidentified-content" + ) + ); + + /** Setup the http request and expectations */ + const req = { body: { calls: [[{ row1: [{ foo: "bar" }] }]] } }; + const res = { + send: (data: Record) => { + expect(data.replies).toEqual([{ row1: "mock-deidentified-content" }]); + done(); + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(dlpClient.deidentifyContent).toHaveBeenCalled(); + expect(debugSpy).toBeCalledWith( + "Incoming request from BigQuery", + req.body.calls + ); + }); + + it("should deidentify data with valid input and a replaceWithInfoType technique and RECORD method", (done) => { + /* Set config data */ + mockedConfig.technique = "replaceWithInfoType"; + mockedConfig.method = "RECORD"; + + /** Configure mock responses */ + mockDeidentityContent.mockImplementation(async () => + createMockDeidentifyContentResponse("table", { + headers: [{ name: "test_header" }], + rows: [{ values: [{ stringValue: "testing" }] }], + }) + ); + + /** Setup the http request and expectations */ + const req = { body: { calls: [[{ row1: [{ foo: "bar" }] }]] } }; + const res = { + send: (data: Record) => { + console.log(data); + expect(data.replies).toEqual([{ test_header: "testing" }]); + done(); + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(dlpClient.deidentifyContent).toHaveBeenCalled(); + expect(debugSpy).toBeCalledWith( + "Incoming request from BigQuery", + req.body.calls + ); + }); + + it("should return invalid method with an unrecognised config method", (done) => { + /* Set config data */ + mockedConfig.technique = undefined; + mockedConfig.method = undefined; + + /** Setup the http request and expectations */ + const req = { body: { calls: [] } }; + const res = { + status: () => { + return { + send: (data: Record) => { + expect(data.errorMessage).toBe("Invalid method"); + done(); + }, + }; + }, + }; + + /** Run run function */ + //@ts-ignore + functions.deidentifyData(req, res); + + /** Check results */ + expect(debugSpy).toBeCalledWith("Incoming request from BigQuery", []); + }); + }); + + describe("reidentifyData", () => { + it("should deidentify data with valid input and a redact technique and INFO_TYPE method", (done) => { + //TODO: add tests if this fn is ready? Currently has now working case in the source code. + expect(true).toBeTruthy(); + done(); + }); + }); + + describe("createBigQueryConnection", () => { + let createBigQueryConnection: any; + + beforeAll(() => { + createBigQueryConnection = testEnv.wrap( + functions.createBigQueryConnection + ); + }); + + it("should successfully create a connection", async () => { + /** Configure mock responses */ + mockCreateConnection.mockImplementation(async () => { + return [ + { + connection: "example", + }, + ]; + }); + + mockCreateQueryJob.mockImplementation(async () => { + return [{ getQueryResults: jest.fn() }]; + }); + + mockSetProcessingState.mockImplementation(); + + /** Run function */ + await createBigQueryConnection({}); + + /** Check results */ + expect(mockCreateConnection).toBeCalledTimes(1); + expect(infoSpy).toBeCalledWith("Connection successfully created 🎉", [ + { + connection: "example", + }, + ]); + + expect(mockSetProcessingState).toBeCalledWith( + "PROCESSING_COMPLETE", + "Connections created successfully." + ); + }); + + it("should wanrn if a conneciton already exists", async () => { + /** Configure mock responses */ + mockCreateConnection.mockImplementation(async () => { + const error = new Error("Connection already exists"); + //@ts-ignore + error.code = 6; + throw error; + }); + + /** Run function */ + await createBigQueryConnection({}); + + /** Check results */ + expect(mockCreateConnection).toBeCalledTimes(1); + expect(warnSpy).toBeCalledWith( + "Connection ext-firestore-geo-functions already exists, will continue creating functions" + ); + }); + + it("should successfully throw an error if a function exists", async () => { + /** Configure mock responses */ + mockCreateConnection.mockImplementation(async () => { + return [ + { + connection: "example", + }, + ]; + }); + + mockCreateQueryJob.mockImplementation(async () => { + const error = new Error("Function already exists"); + //@ts-ignore + error.code = 6; + throw error; + }); + + /** Run function */ + await createBigQueryConnection({}); + + /** Check results */ + expect(mockCreateConnection).toBeCalledTimes(1); + expect(warnSpy).toBeCalledWith("Functions already exists."); + }); + + it("should successfully throw an error if an error code does not equal 6", async () => { + /** Configure mock responses */ + mockCreateConnection.mockImplementation(async () => { + return [ + { + connection: "example", + }, + ]; + }); + + mockCreateQueryJob.mockImplementation(async () => { + const error = new Error("Function already exists"); + //@ts-ignore + error.code = 4; + throw error; + }); + + /** Run function */ + await createBigQueryConnection({}); + + /** Check results */ + expect(mockCreateConnection).toBeCalledTimes(1); + + expect(mockSetProcessingState).toBeCalledWith( + "PROCESSING_FAILED", + "Connections were not created, check logs for more details." + ); + }); + }); +}); diff --git a/bigquery-dlp-functions/functions/__tests__/jest.setup.ts b/bigquery-dlp-functions/functions/__tests__/jest.setup.ts new file mode 100644 index 00000000..d83474ee --- /dev/null +++ b/bigquery-dlp-functions/functions/__tests__/jest.setup.ts @@ -0,0 +1,15 @@ +const path = require("path"); + +(async function () { + require("dotenv").config({ + path: path.resolve( + __dirname, + "../../../_emulator/extensions/firestore-places-autocomplete.env.local" + ), + }); + + process.env.EXT_INSTANCE_ID = "firestore-geo-functions"; + process.env.GCLOUD_PROJECT = "demo-test"; + process.env.PROJECT_ID = "demo-test"; + process.env.EVENTARC_CHANNEL = "my-eventarc-channel"; +})(); diff --git a/bigquery-dlp-functions/functions/__tests__/setupEnvironment.ts b/bigquery-dlp-functions/functions/__tests__/setupEnvironment.ts new file mode 100644 index 00000000..ae8eb1c7 --- /dev/null +++ b/bigquery-dlp-functions/functions/__tests__/setupEnvironment.ts @@ -0,0 +1,8 @@ +export default () => { + process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; + process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "localhost:8080"; + process.env.FIREBASE_AUTH_EMULATOR_HOST = "localhost:9099"; + process.env.PUBSUB_EMULATOR_HOST = "localhost:8085"; + process.env.GOOGLE_CLOUD_PROJECT = "demo-test"; + process.env.FIREBASE_STORAGE_EMULATOR_HOST = "localhost:9199"; +}; diff --git a/bigquery-dlp-functions/functions/__tests__/tsconfig.json b/bigquery-dlp-functions/functions/__tests__/tsconfig.json new file mode 100644 index 00000000..a32dd4f2 --- /dev/null +++ b/bigquery-dlp-functions/functions/__tests__/tsconfig.json @@ -0,0 +1,4 @@ +{ + "extends": "../tsconfig.json", + "include": ["**/*"] +} diff --git a/bigquery-dlp-function/functions/package.json b/bigquery-dlp-functions/functions/package.json similarity index 83% rename from bigquery-dlp-function/functions/package.json rename to bigquery-dlp-functions/functions/package.json index 02528045..23d1b745 100644 --- a/bigquery-dlp-function/functions/package.json +++ b/bigquery-dlp-functions/functions/package.json @@ -17,16 +17,19 @@ "@google-cloud/bigquery-connection": "^2.0.2", "@google-cloud/dlp": "^4.2.0", "firebase-admin": "^11.3.0", - "firebase-functions": "^3.18.0" + "firebase-functions": "^4.2.0" }, "devDependencies": { + "@types/jest": "^29.5.0", "@typescript-eslint/eslint-plugin": "^5.12.0", "@typescript-eslint/parser": "^5.12.0", "eslint": "^8.9.0", "eslint-config-google": "^0.14.0", "eslint-plugin-import": "^2.25.4", - "firebase-functions-test": "^0.2.0", - "typescript": "^4.5.4" + "typescript": "^4.5.4", + "jest": "^29.5.0", + "ts-jest": "^29.0.5", + "firebase-functions-test": "^3.0.0" }, "private": true } diff --git a/bigquery-dlp-function/functions/src/config.ts b/bigquery-dlp-functions/functions/src/config.ts similarity index 100% rename from bigquery-dlp-function/functions/src/config.ts rename to bigquery-dlp-functions/functions/src/config.ts diff --git a/bigquery-dlp-function/functions/src/deidentify.ts b/bigquery-dlp-functions/functions/src/deidentify.ts similarity index 100% rename from bigquery-dlp-function/functions/src/deidentify.ts rename to bigquery-dlp-functions/functions/src/deidentify.ts diff --git a/bigquery-dlp-function/functions/src/index.ts b/bigquery-dlp-functions/functions/src/index.ts similarity index 97% rename from bigquery-dlp-function/functions/src/index.ts rename to bigquery-dlp-functions/functions/src/index.ts index 098bc582..7c1452c3 100644 --- a/bigquery-dlp-function/functions/src/index.ts +++ b/bigquery-dlp-functions/functions/src/index.ts @@ -1,6 +1,6 @@ import * as functions from "firebase-functions"; import * as admin from "firebase-admin"; -import DLP from "@google-cloud/dlp"; +import { DlpServiceClient } from "@google-cloud/dlp"; import { ConnectionServiceClient } from "@google-cloud/bigquery-connection"; import { BigQuery } from "@google-cloud/bigquery"; import { getExtensions } from "firebase-admin/extensions"; @@ -20,10 +20,11 @@ import { reidentifyWithInfoTypeTransformations } from "./reidentify"; admin.initializeApp(); -const dlp = new DLP.DlpServiceClient(); const bigqueryClient = new BigQuery(); const bigqueryConnectionClient = new ConnectionServiceClient(); +const dlp = new DlpServiceClient(); + exports.deidentifyData = functions.https.onRequest( async (request, response) => { const { calls } = request.body; @@ -118,7 +119,7 @@ exports.reidentifyData = functions.https.onRequest( } ); -exports.createBigQueryConnection = functions.tasks +export const createBigQueryConnection = functions.tasks .taskQueue() .onDispatch(async () => { const runtime = getExtensions().runtime(); diff --git a/bigquery-dlp-function/functions/src/reidentify.ts b/bigquery-dlp-functions/functions/src/reidentify.ts similarity index 100% rename from bigquery-dlp-function/functions/src/reidentify.ts rename to bigquery-dlp-functions/functions/src/reidentify.ts diff --git a/bigquery-dlp-function/functions/src/transofmrations.ts b/bigquery-dlp-functions/functions/src/transofmrations.ts similarity index 100% rename from bigquery-dlp-function/functions/src/transofmrations.ts rename to bigquery-dlp-functions/functions/src/transofmrations.ts diff --git a/bigquery-dlp-function/functions/tsconfig.json b/bigquery-dlp-functions/functions/tsconfig.json similarity index 100% rename from bigquery-dlp-function/functions/tsconfig.json rename to bigquery-dlp-functions/functions/tsconfig.json From e34bfd3dc6c07f30eae7f35158b2166ce32c5712 Mon Sep 17 00:00:00 2001 From: Darren Ackers Date: Mon, 24 Apr 2023 20:16:29 +0100 Subject: [PATCH 37/37] docs: re-generate readme --- bigquery-dlp-functions/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/bigquery-dlp-functions/README.md b/bigquery-dlp-functions/README.md index 13795b48..408bc9b7 100644 --- a/bigquery-dlp-functions/README.md +++ b/bigquery-dlp-functions/README.md @@ -2,7 +2,7 @@ **Author**: Firebase (**[https://firebase.google.com](https://firebase.google.com)**) -**Description**: TODO +**Description**: This extension creates BigQuery functions to facilitate de-identification and re-identification in queries, providing configurable techniques, seamless integration, and ensuring better data privacy and compliance. --- @@ -56,6 +56,8 @@ When you use Firebase Extensions, you're only charged for the underlying resourc - DLP Transformation Technique: The technique used by Data Loss Prevention API to deidentify and/or encrypt sensitive information in the data. +- List of fields to transform using record transformation (comma separated): The list of fields to transform using record transformation. This is only used when the transformation method is set to `RECORD`. + - BigQuery Dataset ID: The ID of the dataset where the extension will create a connection. - Cloud Functions location: Where do you want to deploy the functions created for this extension? You usually want a location close to your database. For help selecting a location, refer to the [location selection guide](https://firebase.google.com/docs/functions/locations).