diff --git a/.gitignore b/.gitignore index 2acfe351..f50f680a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ .vscode node_modules/ *.env +*.env.local firebase-debug.log .DS_Store mods-test-data/key.json @@ -11,4 +12,5 @@ build local.properties coverage package-lock.json -yarn.lock \ No newline at end of file +yarn.lock +*.log \ No newline at end of file diff --git a/_emulator/.firebaserc b/_emulator/.firebaserc new file mode 100644 index 00000000..d2ad98e2 --- /dev/null +++ b/_emulator/.firebaserc @@ -0,0 +1,5 @@ +{ + "projects": { + "default": "demo-test" + } +} diff --git a/_emulator/firebase.json b/_emulator/firebase.json new file mode 100644 index 00000000..a0e92533 --- /dev/null +++ b/_emulator/firebase.json @@ -0,0 +1,32 @@ +{ + "extensions": { + "firegraphql-extension": "../firegraphql-extension" + }, + "storage": { + "rules": "storage.rules" + }, + "emulators": { + "hub": { + "port": 4000 + }, + "storage": { + "port": 9199 + }, + "auth": { + "port": 9099 + }, + "pubsub": { + "port": 8085 + }, + "functions": { + "port": 5001 + }, + "ui": { + "enabled": true + } + }, + "firestore": { + "rules": "firestore.rules", + "indexes": "firestore.indexes.json" + } +} diff --git a/_emulator/firestore.indexes.json b/_emulator/firestore.indexes.json new file mode 100644 index 00000000..415027e5 --- /dev/null +++ b/_emulator/firestore.indexes.json @@ -0,0 +1,4 @@ +{ + "indexes": [], + "fieldOverrides": [] +} diff --git a/_emulator/firestore.rules b/_emulator/firestore.rules new file mode 100644 index 00000000..4a44b27d --- /dev/null +++ b/_emulator/firestore.rules @@ -0,0 +1,16 @@ +service cloud.firestore { + match /databases/{database}/documents { + match /{document=**} { + // This rule allows anyone with your database reference to view, edit, + // and delete all data in your database. It is useful for getting + // started, but it is configured to expire after 30 days because it + // leaves your app open to attackers. At that time, all client + // requests to your database will be denied. + // + // Make sure to write security rules for your app before that time, or + // else all client requests to your database will be denied until you + // update your rules. + allow read, write: if true; + } + } +} diff --git a/_emulator/import/auth_export/accounts.json b/_emulator/import/auth_export/accounts.json new file mode 100644 index 00000000..135e08db --- /dev/null +++ b/_emulator/import/auth_export/accounts.json @@ -0,0 +1 @@ +{"kind":"identitytoolkit#DownloadAccountResponse","users":[]} \ No newline at end of file diff --git a/_emulator/import/auth_export/config.json b/_emulator/import/auth_export/config.json new file mode 100644 index 00000000..8f77af98 --- /dev/null +++ b/_emulator/import/auth_export/config.json @@ -0,0 +1 @@ +{"signIn":{"allowDuplicateEmails":false}} \ No newline at end of file diff --git a/_emulator/import/firebase-export-metadata.json b/_emulator/import/firebase-export-metadata.json new file mode 100644 index 00000000..40c53cea --- /dev/null +++ b/_emulator/import/firebase-export-metadata.json @@ -0,0 +1,16 @@ +{ + "version": "11.9.0", + "firestore": { + "version": "1.14.4", + "path": "firestore_export", + "metadata_file": "firestore_export/firestore_export.overall_export_metadata" + }, + "auth": { + "version": "11.9.0", + "path": "auth_export" + }, + "storage": { + "version": "11.9.0", + "path": "storage_export" + } +} \ No newline at end of file diff --git a/_emulator/import/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata b/_emulator/import/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata new file mode 100644 index 00000000..41cea4e8 Binary files /dev/null and b/_emulator/import/firestore_export/all_namespaces/all_kinds/all_namespaces_all_kinds.export_metadata differ diff --git a/_emulator/import/firestore_export/all_namespaces/all_kinds/output-0 b/_emulator/import/firestore_export/all_namespaces/all_kinds/output-0 new file mode 100644 index 00000000..4387d8b8 Binary files /dev/null and b/_emulator/import/firestore_export/all_namespaces/all_kinds/output-0 differ diff --git a/_emulator/import/firestore_export/firestore_export.overall_export_metadata b/_emulator/import/firestore_export/firestore_export.overall_export_metadata new file mode 100644 index 00000000..c3f124a1 Binary files /dev/null and b/_emulator/import/firestore_export/firestore_export.overall_export_metadata differ diff --git a/_emulator/import/storage_export/blobs/c5720248-816c-4ade-b076-ae1daabddfdf b/_emulator/import/storage_export/blobs/c5720248-816c-4ade-b076-ae1daabddfdf new file mode 100644 index 00000000..a7f5f5b1 --- /dev/null +++ b/_emulator/import/storage_export/blobs/c5720248-816c-4ade-b076-ae1daabddfdf @@ -0,0 +1,9 @@ +type Test { + foo: String! +} + +type Query { + testsCollection: [Test]! @firestoreQuery(collection: "tests") + testsCollectionWithCustomId: [Test]! @firestoreQuery(collection: "tests", idField: "customId") + testsDocument(id: ID!): Test @firestoreDoc(collection: "tests") +} \ No newline at end of file diff --git a/_emulator/import/storage_export/buckets.json b/_emulator/import/storage_export/buckets.json new file mode 100644 index 00000000..b49159e6 --- /dev/null +++ b/_emulator/import/storage_export/buckets.json @@ -0,0 +1,10 @@ +{ + "buckets": [ + { + "id": "demo-experimental.appspot.com" + }, + { + "id": "extensions-testing.appspot.com" + } + ] +} \ No newline at end of file diff --git a/_emulator/import/storage_export/metadata/c5720248-816c-4ade-b076-ae1daabddfdf.json b/_emulator/import/storage_export/metadata/c5720248-816c-4ade-b076-ae1daabddfdf.json new file mode 100644 index 00000000..fa1f0c3a --- /dev/null +++ b/_emulator/import/storage_export/metadata/c5720248-816c-4ade-b076-ae1daabddfdf.json @@ -0,0 +1,19 @@ +{ + "name": "firegraphql-ext-schema.gql", + "bucket": "demo-experimental.appspot.com", + "metageneration": 1, + "generation": 1663321798620, + "contentType": "application/octet-stream", + "storageClass": "STANDARD", + "contentDisposition": "inline", + "downloadTokens": [ + "c73185b4-6034-49fd-a436-fb04fe4aaf19" + ], + "etag": "lBh57QypEvVlY77fVvooxkd66T0", + "customMetadata": {}, + "timeCreated": "2022-09-16T09:49:58.620Z", + "updated": "2022-09-16T09:49:58.620Z", + "size": 271, + "md5Hash": "LX9evbwKAcukpur9IKJiOA==", + "crc32c": "340707379" +} \ No newline at end of file diff --git a/_emulator/storage.rules b/_emulator/storage.rules new file mode 100644 index 00000000..d0e06b61 --- /dev/null +++ b/_emulator/storage.rules @@ -0,0 +1,8 @@ +rules_version = '2'; +service firebase.storage { + match /b/{bucket}/o { + match /{allPaths=**} { + allow read, write: if true; + } + } +} \ No newline at end of file diff --git a/firegraphql-extension/.gitignore b/firegraphql-extension/.gitignore new file mode 100644 index 00000000..b3942c4c --- /dev/null +++ b/firegraphql-extension/.gitignore @@ -0,0 +1,3 @@ +test-params.env +firestore-debug.log +ui-debug.log \ No newline at end of file diff --git a/firegraphql-extension/CHANGELOG.md b/firegraphql-extension/CHANGELOG.md new file mode 100644 index 00000000..e69de29b diff --git a/firegraphql-extension/LICENSE b/firegraphql-extension/LICENSE new file mode 100644 index 00000000..d6456956 --- /dev/null +++ b/firegraphql-extension/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/firegraphql-extension/POSTINSTALL.md b/firegraphql-extension/POSTINSTALL.md new file mode 100644 index 00000000..196417c2 --- /dev/null +++ b/firegraphql-extension/POSTINSTALL.md @@ -0,0 +1,9 @@ +### See it in action + + + +### Using the extension + + + +### Monitoring diff --git a/firegraphql-extension/PREINSTALL.md b/firegraphql-extension/PREINSTALL.md new file mode 100644 index 00000000..4c33a5f4 --- /dev/null +++ b/firegraphql-extension/PREINSTALL.md @@ -0,0 +1,8 @@ + + +#### Additional setup + + +#### Billing + + diff --git a/firegraphql-extension/README.md b/firegraphql-extension/README.md new file mode 100644 index 00000000..e69de29b diff --git a/firegraphql-extension/contributing.md b/firegraphql-extension/contributing.md new file mode 100644 index 00000000..f1f7ef73 --- /dev/null +++ b/firegraphql-extension/contributing.md @@ -0,0 +1,1215 @@ + + + + + + + + + + + + + + + + + + + new-project/contributing.md at master · google/new-project + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+ Skip to content + + + + + + + +
+ +
+ +
+ + +
+ + + +
+
+
+ +
+ +
+
+

Learn Git and GitHub without any code!

+

+ Using the Hello World guide, you’ll start a branch, write comments, and open a pull request. +

+ Read the guide +
+
+
+
+ +
+ +
+ +
+

+ + + + / + + new-project + + +

+ +
+ +
    + +
  • +
    + +
    + + + Watch + + + + +
    + Notifications +
    +
    + + + + + + + +
    +     +
    + + +
    +
    • + +
  • +
    +
    + + +
    +
    + + +
    + +
    • + +
  • +
    +
    + +
    • +
+ +
+ + +
+ +
+
+ + Permalink + + + + +
+ +
+ + + master + + + + +
+ + + +
+ +
+ +

+ new-project/docs/contributing.md +

+ + Go to file + + +
+ + + + + + + + +
+ +
+ +
+
+ + @rspier + +
+
+ rspier + + + opensource.google.com => opensource.google + +
+ + + + + +
+
+ + Latest commit + 53a23fe + Sep 27, 2019 + + + + + + History + + +
+
+ +
+ +
+
+ + + 2 + + contributors + + +
+ +

+ Users who have contributed to this file +

+
+ + +
+ + + @grant + + @rspier + + + +
+
+ +
+ +
+
+ + 28 lines (20 sloc) + + 1.07 KB + +
+ +
+ +
+ Raw + Blame +
+ +
+ + + + +
+ + +
+
+ +
+
+
+ +
+

How to Contribute

+

We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow.

+

Contributor License Agreement

+

Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to https://cla.developers.google.com/ to see +your current agreements on file or to sign a new one.

+

You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again.

+

Code reviews

+

All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +GitHub Help for more +information on using pull requests.

+

Community Guidelines

+

This project follows Google's Open Source Community +Guidelines.

+
+
+ +
+ +
+ + +
+ + +
 +
+ +
+
+ +
+ +
+ +
+ + + +
+ + + You can’t perform that action at this time. +
+ + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/firegraphql-extension/extension.yaml b/firegraphql-extension/extension.yaml new file mode 100644 index 00000000..15cc967a --- /dev/null +++ b/firegraphql-extension/extension.yaml @@ -0,0 +1,120 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: firegraphql-extension +version: 0.1.1 +specVersion: v1beta + +displayName: FiregraphQL + +description: >- + Use this extension to set up an endpoint to make GraphQL queries against resources on your Firebase project (Firebase Firestore, RTDB and Storage). + +license: Apache-2.0 + +sourceUrl: https://github.com/FirebaseExtended/experimental-extensions + +author: + authorName: Firebase + url: https://firebase.google.com + +contributors: + - authorName: Invertase + email: oss@invertase.io + url: https://github.com/invertase + +billingRequired: true + +roles: + - role: storage.objectViewer + reason: Allows the extension to download the GraphQL Schema file from the Cloud Storage bucket. + - role: firebaseauth.viewer + reason: Allows the extension to get the project Web App credentials. + +resources: + - name: executeQuery + type: firebaseextensions.v1beta.function + properties: + location: ${LOCATION} + httpsTrigger: {} + runtime: "nodejs14" + +params: + - param: LOCATION + label: Cloud Functions location + description: >- + Where do you want to deploy the functions created for this extension? + For help selecting a location, refer to the [location selection + guide](https://firebase.google.com/docs/functions/locations). + type: select + options: + - label: Iowa (us-central1) + value: us-central1 + - label: South Carolina (us-east1) + value: us-east1 + - label: Northern Virginia (us-east4) + value: us-east4 + - label: Belgium (europe-west1) + value: europe-west1 + - label: London (europe-west2) + value: europe-west2 + - label: Frankfurt (europe-west3) + value: europe-west3 + - label: Hong Kong (asia-east2) + value: asia-east2 + - label: Tokyo (asia-northeast1) + value: asia-northeast1 + default: us-central1 + required: true + immutable: true + + - param: STORAGE_BUCKET + label: Cloud Storage bucket for GQL schemas + description: > + To which Cloud Storage bucket will you upload the GQL schemas? + type: selectResource + resourceType: storage.googleapis.com/Bucket + example: my-project-12345.appspot.com + validationRegex: ^([0-9a-z_.-]*)$ + validationErrorMessage: Invalid storage bucket + default: ${STORAGE_BUCKET} + required: true + + - param: SCHEMA_PATH + label: GQL Schema path + description: > + A path to a file in Cloud Storage representing the GraphQL schema. + required: true + + - param: FIREBASE_WEB_APP_ID + label: Firebase Web App ID + description: > + The ID of the Firebase web app which will be used to make client requests. If not provided, the extension will attempt to use + one of the Firebase web apps in your project. + example: 1:11111111111:web:xxxxxxxxxxxxxxxxxxxxxx + required: false + + - param: FIREBASE_APPS_LRU_MAX + label: Max Firebase Apps LRU Cache Size + description: > + The maximum number of Firebase app instances to store in the LRU cache. + example: 100 + required: false + + - param: FIREBASE_APPS_LRU_TTL + label: Firebase Apps LRU Cache TTL + description: > + The number of milliseconds to keep Firebase app instances in the LRU cache. + example: 600000 + required: false diff --git a/firegraphql-extension/functions/.gitignore b/firegraphql-extension/functions/.gitignore new file mode 100644 index 00000000..aa0b6b37 --- /dev/null +++ b/firegraphql-extension/functions/.gitignore @@ -0,0 +1,6 @@ +# Typescript v1 declaration files +typings/ + +node_modules/ + +lib/ \ No newline at end of file diff --git a/firegraphql-extension/functions/__tests__/functions.test.ts b/firegraphql-extension/functions/__tests__/functions.test.ts new file mode 100644 index 00000000..6f871861 --- /dev/null +++ b/firegraphql-extension/functions/__tests__/functions.test.ts @@ -0,0 +1,82 @@ +/** + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; +import fetch from "node-fetch"; + +process.env.FIRESTORE_EMULATOR_HOST = "localhost:8080"; +process.env.FIREBASE_FIRESTORE_EMULATOR_ADDRESS = "localhost:8080"; +process.env.FIREBASE_AUTH_EMULATOR_HOST = "localhost:9099"; +process.env.FIREBASE_STORAGE_EMULATOR_HOST = "localhost:9199"; +process.env.FIREBASE_DATABASE_EMULATOR_HOST = "localhost:9000"; +process.env.PUBSUB_EMULATOR_HOST = "localhost:8085"; +process.env.GOOGLE_CLOUD_PROJECT = "demo-experimental"; + +if (admin.apps.length === 0) { + admin.initializeApp({ projectId: "demo-experimental" }); +} + +const extName = "ext-firegraphql-extension-executeQuery"; +const url = `http://0.0.0.0:5001/demo-experimental/us-central1/${extName}`; + +async function postReq(query: string, variables?: any) { + const res = await fetch(url, { + method: "POST", + headers: { + "Content-Type": "application/json", + }, + body: JSON.stringify({ + query, + variables, + }), + }); + + return await res.json(); +} + +describe("FiregraphQL Extension", () => { + it("returns collection data", async () => { + const res = await postReq(`query { testsCollection }`); + + expect(res.errors).toBeUndefined(); + expect(res.data.testsCollection.length).toBe(3); + + res.data.testsCollection.forEach((doc: any) => { + expect(doc.foo).toBeDefined(); + expect(doc.id).toBeDefined(); + }); + }); + + it("returns collection data with a custom id", async () => { + const res = await postReq(`query { testsCollectionWithCustomId }`); + + expect(res.errors).toBeUndefined(); + expect(res.data.testsCollectionWithCustomId.length).toBe(3); + + res.data.testsCollectionWithCustomId.forEach((doc: any) => { + expect(doc.foo).toBeDefined(); + expect(doc.customId).toBeDefined(); + }); + }); + + it("returns a specific document", async () => { + const res = await postReq(`query { testsDocument(id: "9vg0QuyugnYAtNrxziiP") }`); + + expect(res.errors).toBeUndefined(); + expect(res.data.testsDocument.foo).toBeDefined(); + expect(res.data.testsDocument.id).toBeDefined(); + }); +}); diff --git a/firegraphql-extension/functions/__tests__/tsconfig.json b/firegraphql-extension/functions/__tests__/tsconfig.json new file mode 100644 index 00000000..cd1fe18a --- /dev/null +++ b/firegraphql-extension/functions/__tests__/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "lib" + }, + "include": ["**/*"] +} diff --git a/firegraphql-extension/functions/firegraphql-0.0.2.tgz b/firegraphql-extension/functions/firegraphql-0.0.2.tgz new file mode 100644 index 00000000..63c4a250 Binary files /dev/null and b/firegraphql-extension/functions/firegraphql-0.0.2.tgz differ diff --git a/firestore-shorten-urls-dynamic-links/functions/lib/config.js b/firegraphql-extension/functions/jest.config.js similarity index 50% rename from firestore-shorten-urls-dynamic-links/functions/lib/config.js rename to firegraphql-extension/functions/jest.config.js index 75fe8b51..5aea3dff 100644 --- a/firestore-shorten-urls-dynamic-links/functions/lib/config.js +++ b/firegraphql-extension/functions/jest.config.js @@ -1,12 +1,11 @@ -"use strict"; -/* - * Copyright 2019 Google LLC +/** + * Copyright 2020 Google LLC * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * - * https://www.apache.org/licenses/LICENSE-2.0 + * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, @@ -14,12 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -Object.defineProperty(exports, "__esModule", { value: true }); -exports.default = { - dynamicLinkUrlPrefix: process.env.DYNAMIC_LINKS_URL_PREFIX, - dynamicLinkSuffixLength: process.env.DYNAMIC_LINKS_SUFFIX_LENGTH, - collectionPath: process.env.COLLECTION_PATH, - location: process.env.LOCATION, - shortUrlFieldName: process.env.SHORT_URL_FIELD_NAME, - urlFieldName: process.env.URL_FIELD_NAME, + +module.exports = { + preset: "ts-jest", + testEnvironment: "node", + testMatch: ["**/__tests__/*.test.ts"], }; diff --git a/firegraphql-extension/functions/package.json b/firegraphql-extension/functions/package.json new file mode 100644 index 00000000..8f0a9d1b --- /dev/null +++ b/firegraphql-extension/functions/package.json @@ -0,0 +1,36 @@ +{ + "name": "firestore-graphql", + "description": "TODO", + "main": "lib/index.js", + "license": "Apache-2.0", + "scripts": { + "prepare": "npm run build", + "build": "npm run clean && npm run compile", + "format": "prettier --write \"**/*.{md,yml,ts,json,yaml}\"", + "lint": "prettier --list-different \"**/*.{md,yml,ts,json,yaml}\"", + "clean": "rimraf lib", + "compile": "tsc", + "test": "jest", + "generate-readme": "firebase ext:info .. --markdown > ../README.md" + }, + "dependencies": { + "firebase": "^9.9.4", + "firebase-admin": "^11.0.1", + "firebase-functions": "^3.6.1", + "firegraphql": "file:firegraphql-0.0.2.tgz", + "googleapis": "^107.0.0", + "rimraf": "^3.0.2", + "typescript": "^4.6.4" + }, + "devDependencies": { + "@types/jest": "^26.0.4", + "@types/node-fetch": "^2.6.2", + "axios": "^0.27.2", + "jest": "^29.0.3", + "mocked-env": "^1.3.2", + "node-fetch": "^2.6.7", + "prettier": "1.15.3", + "ts-jest": "^29.0.1" + }, + "private": true +} diff --git a/firegraphql-extension/functions/src/api.ts b/firegraphql-extension/functions/src/api.ts new file mode 100644 index 00000000..68b1b107 --- /dev/null +++ b/firegraphql-extension/functions/src/api.ts @@ -0,0 +1,50 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FirebaseOptions } from "firebase/app"; +import { google } from "googleapis"; +import config from "./config"; + +const firebase = google.firebase("v1beta1"); + +const auth = new google.auth.GoogleAuth({ + scopes: ["https://www.googleapis.com/auth/firebase.readonly"], +}); + +export async function getWebConfigByAppId( + appId: string +): Promise { + const res = await firebase.projects.webApps.getConfig({ + auth: await auth.getClient(), + name: `projects/${config.projectId}/webApps/${appId}/config`, + }); + + return res.data; +} + +export async function getWebConfigByList(): Promise { + const res = await firebase.projects.webApps.list({ + auth: await auth.getClient(), + pageSize: 1, + parent: `projects/${config.projectId}`, + }); + + if (res.data.apps.length === 0) { + throw Error("This Firebase project has no Web Applications configured. Please visit https://firebase.google.com/docs/web/setup#register-app to create one."); + } + + return res.data.apps[0]; +} diff --git a/firegraphql-extension/functions/src/config.ts b/firegraphql-extension/functions/src/config.ts new file mode 100644 index 00000000..23f5e5b3 --- /dev/null +++ b/firegraphql-extension/functions/src/config.ts @@ -0,0 +1,43 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { FirebaseOptions } from "firebase/app"; + +export default { + projectId: process.env.PROJECT_ID, + location: process.env.LOCATION, + webAppId: process.env.FIREBASE_WEB_APP_ID, + storageBucket: process.env.STORAGE_BUCKET, + schemaPath: process.env.SCHEMA_PATH, + firebaseOptions: parseFirebaseOptions(process.env.FIREBASE_OPTIONS), + firestoreEmulator: parseEmulatorHost(process.env.FIRESTORE_EMULATOR_HOST), + databaseEmulator: parseEmulatorHost(process.env.DATABASE_EMULATOR_HOST), + storageEmulator: parseEmulatorHost(process.env.STORAGE_EMULATOR_HOST), +}; + +function parseEmulatorHost(uri: string): { host: string; port: number } | undefined { + if (!uri) return undefined; + const url = new URL(uri.startsWith('http') ? uri : `http://${uri}`); + return { host: url.hostname, port: Number(url.port) }; +} + +function parseFirebaseOptions(jsonString: string): FirebaseOptions | undefined { + try { + return JSON.parse(jsonString); + } catch (e) { + return undefined; + } +} diff --git a/firegraphql-extension/functions/src/index.ts b/firegraphql-extension/functions/src/index.ts new file mode 100644 index 00000000..0071a418 --- /dev/null +++ b/firegraphql-extension/functions/src/index.ts @@ -0,0 +1,89 @@ +/* + * Copyright 2020 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import * as admin from "firebase-admin"; +import * as functions from "firebase-functions"; +import { FirebaseApolloFunction, makeExecutableSchema } from "firegraphql"; + +import config from "./config"; +import * as logs from "./logs"; + +import { getWebConfigByAppId, getWebConfigByList } from "./api"; + +const app = admin.initializeApp(); +let server: FirebaseApolloFunction; + +logs.init(); + +export const executeQuery = functions.handler.https.onRequest( + async (req, res) => { + if (!server) { + let firebaseOptions = config.firebaseOptions; + let typeDef: string; + + // During integration tests, we can't mock this call to GoogleApis easily, + // and since tests are running in the emulator, options are not needed. + if (!firebaseOptions) { + try { + if (config.webAppId) { + firebaseOptions = await getWebConfigByAppId(config.webAppId); + } else { + firebaseOptions = await getWebConfigByList(); + } + } catch (e: any) { + logs.webConfigError(e); + return res.status(400).send(e?.message ?? e ?? '400 Bad Request'); + } + } + + try { + typeDef = await downloadSchema(); + } catch (e) { + logs.downloadSchemaError(e); + return res.status(400).send(e?.message ?? e ?? '400 Bad Request'); + } + + server = new FirebaseApolloFunction({ + schema: makeExecutableSchema({ + typeDefs: [typeDef], + }), + options: { + firebaseAdminAppInstance: app, + firebaseOptions, + connectFirestoreEmulator: config.firestoreEmulator, + connectDatabaseEmulator: config.databaseEmulator, + connectStorageEmulator: config.storageEmulator, + }, + }); + } + + return server.createHandler()(req, res); + } +); + +// Downloads a file from the storage bucket at the provided ref. +async function downloadSchema(): Promise { + const download = await app + .storage() + .bucket(config.storageBucket) + .file(config.schemaPath) + .download({ + // https://github.com/googleapis/google-cloud-node/issues/654#issuecomment-987123067 + validation: !process.env.FUNCTIONS_EMULATOR, + }); + + return download.toString(); +} diff --git a/firegraphql-extension/functions/src/logs.ts b/firegraphql-extension/functions/src/logs.ts new file mode 100644 index 00000000..a145567b --- /dev/null +++ b/firegraphql-extension/functions/src/logs.ts @@ -0,0 +1,51 @@ +/* + * Copyright 2022 Google LLC + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { logger } from "firebase-functions"; +import config from "./config"; + +export const obfuscatedConfig = { + ...config, +}; + +export const init = () => { + logger.log("Initializing extension with configuration", obfuscatedConfig); +}; + +export const webConfigError = (err: Error) => { + if (config.webAppId) { + logger.error( + `Error fetching web config, do you have access to the Web App with ID ${ + config.webAppId + }?`, + err + ); + } else { + logger.error( + `Error fetching a Web App configuration, have you created one?`, + err + ); + } +}; + +export const downloadSchemaError = (e: Error) => { + logger.error( + `Error downloading schema on Bucket "${config.storageBucket}" at path "${ + config.schemaPath + }"`, + e + ); +}; diff --git a/firegraphql-extension/functions/tsconfig.json b/firegraphql-extension/functions/tsconfig.json new file mode 100644 index 00000000..1915eff3 --- /dev/null +++ b/firegraphql-extension/functions/tsconfig.json @@ -0,0 +1,16 @@ +{ + "compilerOptions": { + "lib": ["esnext.asynciterable", "es2020", "es6"], + "outDir": "lib", + "module": "commonjs", + "noImplicitReturns": true, + "sourceMap": false, + "target": "es6", + "esModuleInterop": true, + "skipLibCheck": true + }, + "compileOnSave": true, + "include": ["src"], + "exclude": ["node_modules"] +} + diff --git a/firestore-shorten-urls-dynamic-links/functions/lib/abstract-shortener.js b/firestore-shorten-urls-dynamic-links/functions/lib/abstract-shortener.js deleted file mode 100644 index a8b3d4d5..00000000 --- a/firestore-shorten-urls-dynamic-links/functions/lib/abstract-shortener.js +++ /dev/null @@ -1,105 +0,0 @@ -"use strict"; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.FirestoreUrlShortener = void 0; -const admin = require("firebase-admin"); -const logs = require("./logs"); -class FirestoreUrlShortener { - constructor(urlFieldName, shortUrlFieldName) { - this.urlFieldName = urlFieldName; - this.shortUrlFieldName = shortUrlFieldName; - this.logs = logs; - this.urlFieldName = urlFieldName; - this.shortUrlFieldName = shortUrlFieldName; - // Initialize the Firebase Admin SDK - admin.initializeApp(); - } - onDocumentCreate(snapshot) { - return __awaiter(this, void 0, void 0, function* () { - this.logs.start(); - if (this.urlFieldName === this.shortUrlFieldName) { - this.logs.fieldNamesNotDifferent(); - return; - } - yield this.handleCreateDocument(snapshot); - this.logs.complete(); - }); - } - onDocumentUpdate(change) { - return __awaiter(this, void 0, void 0, function* () { - this.logs.start(); - if (this.urlFieldName === this.shortUrlFieldName) { - this.logs.fieldNamesNotDifferent(); - return; - } - yield this.handleUpdateDocument(change.before, change.after); - this.logs.complete(); - }); - } - extractUrl(snapshot) { - return snapshot.get(this.urlFieldName); - } - handleCreateDocument(snapshot) { - return __awaiter(this, void 0, void 0, function* () { - const url = this.extractUrl(snapshot); - if (url) { - this.logs.documentCreatedWithUrl(); - yield this.shortenUrl(snapshot); - } - else { - this.logs.documentCreatedNoUrl(); - } - }); - } - handleUpdateDocument(before, after) { - return __awaiter(this, void 0, void 0, function* () { - const urlAfter = this.extractUrl(after); - const urlBefore = this.extractUrl(before); - if (urlAfter === urlBefore) { - this.logs.documentUpdatedUnchangedUrl(); - } - else if (urlAfter) { - this.logs.documentUpdatedChangedUrl(); - yield this.shortenUrl(after); - } - else if (urlBefore) { - this.logs.documentUpdatedDeletedUrl(); - yield this.updateShortUrl(after, admin.firestore.FieldValue.delete()); - } - else { - this.logs.documentUpdatedNoUrl(); - } - }); - } - updateShortUrl(snapshot, url) { - return __awaiter(this, void 0, void 0, function* () { - this.logs.updateDocument(snapshot.ref.path); - yield snapshot.ref.update(this.shortUrlFieldName, url); - this.logs.updateDocumentComplete(snapshot.ref.path); - }); - } -} -exports.FirestoreUrlShortener = FirestoreUrlShortener; diff --git a/firestore-shorten-urls-dynamic-links/functions/lib/index.js b/firestore-shorten-urls-dynamic-links/functions/lib/index.js deleted file mode 100644 index a0069e77..00000000 --- a/firestore-shorten-urls-dynamic-links/functions/lib/index.js +++ /dev/null @@ -1,112 +0,0 @@ -"use strict"; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.shorten_update = exports.shorten_create = void 0; -const url_1 = require("url"); -const functions = require("firebase-functions"); -const node_fetch_1 = require("node-fetch"); -const abstract_shortener_1 = require("./abstract-shortener"); -const config_1 = require("./config"); -const logs = require("./logs"); -class ServiceAccountCredential { - constructor() { - this.metadataServiceUri = new url_1.URL("http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token"); - this.requiredScopes = "https://www.googleapis.com/auth/firebase"; - const searchParams = new url_1.URLSearchParams({ scopes: this.requiredScopes }); - this.metadataServiceUri.search = searchParams.toString(); - } - getAccessToken() { - return __awaiter(this, void 0, void 0, function* () { - const now = Math.floor(Date.now() / 1000); - const nowish = now + 30; // 30s leeway - if (typeof this.accessToken === "undefined" || - typeof this.tokenExpiration === "undefined" || - this.tokenExpiration < nowish) { - const metadataResponse = yield (0, node_fetch_1.default)(this.metadataServiceUri, { - headers: { - "Metadata-Flavor": "Google", - }, - }); - const { access_token, expires_in } = yield metadataResponse.json(); - this.accessToken = access_token; - this.tokenExpiration = now + expires_in; - } - return this.accessToken; - }); - } -} -class FirestoreDynamicLinksUrlShortener extends abstract_shortener_1.FirestoreUrlShortener { - constructor(urlFieldName, shortUrlFieldName, dynamicLinkUrlPrefix, dynamicLinkSuffixLength) { - super(urlFieldName, shortUrlFieldName); - this.dynamicLinkUrlPrefix = dynamicLinkUrlPrefix; - this.dynamicLinkSuffixLength = dynamicLinkSuffixLength; - this.dynamicLinksApiUrl = "https://firebasedynamiclinks.googleapis.com/v1/shortLinks"; - this.credentials = new ServiceAccountCredential(); - this.logs = logs; - this.logs.init(); - } - shortenUrl(snapshot) { - return __awaiter(this, void 0, void 0, function* () { - const url = this.extractUrl(snapshot); - this.logs.shortenUrl(url); - const accessToken = yield this.credentials.getAccessToken(); - try { - const requestBody = { - dynamicLinkInfo: { - domainUriPrefix: this.dynamicLinkUrlPrefix, - link: url, - }, - suffix: { option: this.dynamicLinkSuffixLength }, - }; - const response = yield (0, node_fetch_1.default)(this.dynamicLinksApiUrl, { - headers: { - Authorization: `Bearer ${accessToken}`, - }, - method: "POST", - body: JSON.stringify(requestBody), - }); - const responseBody = yield response.json(); - if (!response.ok) { - this.logs.error(responseBody.error.message); - return; - } - const shortUrl = responseBody.shortLink; - this.logs.shortenUrlComplete(shortUrl); - yield this.updateShortUrl(snapshot, shortUrl); - } - catch (err) { - this.logs.error(err.body); - } - }); - } -} -const urlShortener = new FirestoreDynamicLinksUrlShortener(config_1.default.urlFieldName, config_1.default.shortUrlFieldName, config_1.default.dynamicLinkUrlPrefix, config_1.default.dynamicLinkSuffixLength); -exports.shorten_create = functions.handler.firestore.document.onCreate((snapshot) => __awaiter(void 0, void 0, void 0, function* () { - return urlShortener.onDocumentCreate(snapshot); -})); -exports.shorten_update = functions.handler.firestore.document.onUpdate((change) => __awaiter(void 0, void 0, void 0, function* () { - return urlShortener.onDocumentUpdate(change); -})); diff --git a/firestore-shorten-urls-dynamic-links/functions/lib/logs.js b/firestore-shorten-urls-dynamic-links/functions/lib/logs.js deleted file mode 100644 index e57e9479..00000000 --- a/firestore-shorten-urls-dynamic-links/functions/lib/logs.js +++ /dev/null @@ -1,80 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.updateDocumentComplete = exports.updateDocument = exports.start = exports.shortenUrlComplete = exports.shortenUrl = exports.init = exports.fieldNamesNotDifferent = exports.error = exports.documentUpdatedUnchangedUrl = exports.documentUpdatedNoUrl = exports.documentUpdatedDeletedUrl = exports.documentUpdatedChangedUrl = exports.documentCreatedWithUrl = exports.documentCreatedNoUrl = exports.complete = void 0; -/* - * Copyright 2019 Google LLC - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * https://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -const functions = require("firebase-functions"); -const config_1 = require("./config"); -const complete = () => { - functions.logger.log("Completed execution of extension"); -}; -exports.complete = complete; -const documentCreatedNoUrl = () => { - functions.logger.log("Document was created without a URL, so no processing is required"); -}; -exports.documentCreatedNoUrl = documentCreatedNoUrl; -const documentCreatedWithUrl = () => { - functions.logger.log("Document was created with a URL"); -}; -exports.documentCreatedWithUrl = documentCreatedWithUrl; -const documentUpdatedChangedUrl = () => { - functions.logger.log("Document was updated: URL has changed"); -}; -exports.documentUpdatedChangedUrl = documentUpdatedChangedUrl; -const documentUpdatedDeletedUrl = () => { - functions.logger.log("Document was updated: URL was deleted"); -}; -exports.documentUpdatedDeletedUrl = documentUpdatedDeletedUrl; -const documentUpdatedNoUrl = () => { - functions.logger.log("Document was updated: no URL exists, so no processing is required"); -}; -exports.documentUpdatedNoUrl = documentUpdatedNoUrl; -const documentUpdatedUnchangedUrl = () => { - functions.logger.log("Document was updated: URL has not changed, so no processing is required"); -}; -exports.documentUpdatedUnchangedUrl = documentUpdatedUnchangedUrl; -const error = (err) => { - functions.logger.error("Error when shortening URL", err); -}; -exports.error = error; -const fieldNamesNotDifferent = () => { - functions.logger.error("The `URL` and `Short URL` field names must be different"); -}; -exports.fieldNamesNotDifferent = fieldNamesNotDifferent; -const init = () => { - functions.logger.log("Initializing extension with configuration", config_1.default); -}; -exports.init = init; -const shortenUrl = (url) => { - functions.logger.log(`Shortening URL: '${url}'`); -}; -exports.shortenUrl = shortenUrl; -const shortenUrlComplete = (shortUrl) => { - functions.logger.log(`Finished shortening URL to: '${shortUrl}'`); -}; -exports.shortenUrlComplete = shortenUrlComplete; -const start = () => { - functions.logger.log("Started execution of extension with configuration", config_1.default); -}; -exports.start = start; -const updateDocument = (path) => { - functions.logger.log(`Updating Cloud Firestore document: '${path}'`); -}; -exports.updateDocument = updateDocument; -const updateDocumentComplete = (path) => { - functions.logger.log(`Finished updating Cloud Firestore document: '${path}'`); -}; -exports.updateDocumentComplete = updateDocumentComplete; diff --git a/package.json b/package.json index 32f8fdf4..45369186 100644 --- a/package.json +++ b/package.json @@ -22,11 +22,13 @@ "url": "https://github.com/FirebaseExtended/experimental-extensions/issues" }, "devDependencies": { + "@types/jest": "^29.0.3", "firebase-tools": "^9.9.0", "husky": "^7.0.4", "lerna": "^3.4.3", "lint-staged": "^12.4.0", "prettier": "^2.2.1", + "ts-jest": "^29.0.1", "typescript": "^4.1.3" }, "lint-staged": { diff --git a/storage-mirror-firestore/functions/lib/config.js b/storage-mirror-firestore/functions/lib/config.js deleted file mode 100644 index d27971b9..00000000 --- a/storage-mirror-firestore/functions/lib/config.js +++ /dev/null @@ -1,15 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -const config = { - bucket: process.env.BUCKET, - location: process.env.LOCATION, - firestoreRoot: process.env.FIRESTORE_ROOT, - itemsSubcollectionName: process.env.ITEMS_SUBCOLLECTION_NAME || "items", - itemsTombstoneSubcollectionName: process.env.ITEMS_TOMBSTONES_NAME || "items-tombstones", - prefixesSubcollectionName: process.env.PREFIXES_SUBCOLLECTION_NAME || "prefixes", - prefixesTombstoneSubcollectionName: process.env.PREFIXES_TOMBSTONES_NAME || "prefixes-tombstones", - customMetadataFieldFilter: new RegExp(process.env.CUSTOM_METADATA_FILTER || ".*"), - metadataFieldFilter: new RegExp(process.env.METADATA_FIELD_FILTER || ".*"), - objectNameFilter: new RegExp(process.env.OBJECT_NAME_FILTER || ".*"), -}; -exports.default = config; diff --git a/storage-mirror-firestore/functions/lib/constants.js b/storage-mirror-firestore/functions/lib/constants.js deleted file mode 100644 index eca4df25..00000000 --- a/storage-mirror-firestore/functions/lib/constants.js +++ /dev/null @@ -1,21 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.Constants = void 0; -class Constants { -} -exports.Constants = Constants; -// Extension Firestore Document Fields -Constants.lastEventField = "lastEvent"; -Constants.childRefField = "childRef"; -Constants.gcsMetadataField = "gcsMetadata"; -// GCS Object Metadata Fields -Constants.objectCustomMetadataField = "metadata"; -// Extension Constants -Constants.queryLimit = 5; -Constants.transactionAttempts = 5; -Constants.numberFields = ["size", "generation", "metageneration"]; -Constants.dateFields = [ - "timeCreated", - "timeStorageClassUpdated", - "updated", -]; diff --git a/storage-mirror-firestore/functions/lib/index.js b/storage-mirror-firestore/functions/lib/index.js deleted file mode 100644 index 67dd69e2..00000000 --- a/storage-mirror-firestore/functions/lib/index.js +++ /dev/null @@ -1,43 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.mirrorObjectPathHttp = exports.mirrorArchive = exports.mirrorDelete = exports.mirrorMetadataUpdate = exports.mirrorFinalize = void 0; -const admin = require("firebase-admin"); -const functions = require("firebase-functions"); -const logs = require("./logs"); -const mirror_1 = require("./mirror"); -// Initialize the Firebase Admin SDK -admin.initializeApp(); -const genericHandler = (object, context) => __awaiter(void 0, void 0, void 0, function* () { - logs.start(); - logs.metadata(object); - logs.context(context); - yield (0, mirror_1.onObjectChange)(object, context.eventType); -}); -exports.mirrorFinalize = functions.storage - .object() - .onFinalize(genericHandler); -exports.mirrorMetadataUpdate = functions.storage - .object() - .onMetadataUpdate(genericHandler); -exports.mirrorDelete = functions.storage.object().onDelete(genericHandler); -exports.mirrorArchive = functions.storage - .object() - .onArchive(genericHandler); -exports.mirrorObjectPathHttp = functions.handler.https.onRequest((req, res) => __awaiter(void 0, void 0, void 0, function* () { - if (req.method != "POST") { - res.status(403).send("Forbidden!"); - return; - } - const path = req.body.path; - yield (0, mirror_1.mirrorObjectPath)(path); - res.sendStatus(200); -})); diff --git a/storage-mirror-firestore/functions/lib/logs.js b/storage-mirror-firestore/functions/lib/logs.js deleted file mode 100644 index ca023cdf..00000000 --- a/storage-mirror-firestore/functions/lib/logs.js +++ /dev/null @@ -1,104 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.missingLastEventField = exports.invalidPrefixDocument = exports.transactionDeleteAttempt = exports.transactionWriteAttempt = exports.transactionReads = exports.context = exports.invalidObjectName = exports.skippedMissingPath = exports.skippedObject = exports.skippedOverwrite = exports.transactionFailure = exports.docDoesNotExist = exports.eventNotStale = exports.staleTieBreak = exports.abortingStaleEvent = exports.startingTransaction = exports.metadata = exports.startHttpEvent = exports.start = exports.error = void 0; -const config_1 = require("./config"); -const constants_1 = require("./constants"); -const error = (err, action) => { - const message = action ? `Error while ${action}:` : "Error:"; - console.error(message, err); -}; -exports.error = error; -const start = () => { - console.log("Started Function execution with Configuration:", config_1.default); -}; -exports.start = start; -const startHttpEvent = () => { - console.log("Started Http Function execution with Configuration:", config_1.default); -}; -exports.startHttpEvent = startHttpEvent; -const metadata = (m) => { - console.log(`Event triggered with Object metadata: ${JSON.stringify(m)}`); -}; -exports.metadata = metadata; -const startingTransaction = (path, attemptNo) => { - console.log(`Starting Transaction attempt number ${attemptNo} for Item Document: ${path}`); -}; -exports.startingTransaction = startingTransaction; -const abortingStaleEvent = (path, eventTime, existingTime) => { - console.log("Aborting update of " + - path + - " because incoming timestamp (" + - eventTime + - ") is older than existing timestamp (" + - existingTime + - ")"); -}; -exports.abortingStaleEvent = abortingStaleEvent; -const staleTieBreak = (path, eventTime, existingTime) => { - console.log("Aborting update of " + - path + - " because incoming timestamp (" + - eventTime + - ") is equal to existing timestamp (" + - existingTime + - ")"); -}; -exports.staleTieBreak = staleTieBreak; -const eventNotStale = (path, eventTime, existingTime) => { - console.log("Update for " + - path + - "is stale because incoming timestamp (" + - eventTime + - ") is newer than existing timestamp (" + - existingTime + - ")"); -}; -exports.eventNotStale = eventNotStale; -const docDoesNotExist = (path) => { - console.log(`${path} is not stale because it does not yet exist.`); -}; -exports.docDoesNotExist = docDoesNotExist; -const transactionFailure = (path, reason) => { - console.error(`Transaction to update ${path} failed because:`, reason); -}; -exports.transactionFailure = transactionFailure; -const skippedOverwrite = (name, eventType) => { - console.log(`Skipped write for ${eventType} event on ${name} because the object exists. This was likely an overwrite of the object.`); -}; -exports.skippedOverwrite = skippedOverwrite; -const skippedObject = (name) => { - console.log(`Skipping Object with name not matching filter: ${name}`); -}; -exports.skippedObject = skippedObject; -const skippedMissingPath = (name) => { - console.log(`Skipping mirroring of path "${name}" because no data was found in Firestore or GCS`); -}; -exports.skippedMissingPath = skippedMissingPath; -const invalidObjectName = (name) => { - console.log(`Skipping Object with invalid name: ${name}`); -}; -exports.invalidObjectName = invalidObjectName; -const context = (m) => { - console.log("Event Context", m); -}; -exports.context = context; -const transactionReads = (paths) => { - console.log(`This transaction read ${paths.length} documents: ${paths}`); -}; -exports.transactionReads = transactionReads; -const transactionWriteAttempt = (num, paths) => { - console.log(`Attempting to write ${num} documents: ${paths}`); -}; -exports.transactionWriteAttempt = transactionWriteAttempt; -const transactionDeleteAttempt = (num, paths) => { - console.log(`Attempting to delete ${num} documents: ${paths}`); -}; -exports.transactionDeleteAttempt = transactionDeleteAttempt; -const invalidPrefixDocument = (id) => { - console.log(`Missing fields in Prefix Document with id: ${id} treating the Document as requiring deletion.`); -}; -exports.invalidPrefixDocument = invalidPrefixDocument; -const missingLastEventField = (id) => { - console.log(`Missing ${constants_1.Constants.lastEventField} field in Document with id: ${id}, treating as stale.`); -}; -exports.missingLastEventField = missingLastEventField; diff --git a/storage-mirror-firestore/functions/lib/mirror.js b/storage-mirror-firestore/functions/lib/mirror.js deleted file mode 100644 index c9859715..00000000 --- a/storage-mirror-firestore/functions/lib/mirror.js +++ /dev/null @@ -1,403 +0,0 @@ -"use strict"; -var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) { - function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); } - return new (P || (P = Promise))(function (resolve, reject) { - function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } } - function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } } - function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); } - step((generator = generator.apply(thisArg, _arguments || [])).next()); - }); -}; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.isStaleEvent = exports.onObjectChange = exports.mirrorObjectPath = void 0; -const firestore_1 = require("@google-cloud/firestore"); -const admin = require("firebase-admin"); -const config_1 = require("./config"); -const constants_1 = require("./constants"); -const logs = require("./logs"); -const util_1 = require("./util"); -/** - * Get the metadata for an object, returning undefined if the object does not exist. - * This is a wrapper around `getMetadata` that returns undefined if the error is a 404. - * - * @param objectName Path to the GCS object (without the bucket) to get metadata for. - */ -function getCurrentMetadata(objectName) { - return __awaiter(this, void 0, void 0, function* () { - try { - const [gcsMetadata, _] = yield admin - .storage() - .bucket(config_1.default.bucket) - .file(objectName) - .getMetadata(); - return firestoreDocumentData(gcsMetadata, "google.storage.object.finalize"); - } - catch (e) { - if (e.code === 404) { - return undefined; - } - else { - logs.error(e, `getting metadata for ${objectName}`); - throw e; - } - } - }); -} -/** - * Mirror the gcs state of the given object to Firestore. - * This works by reading from GCS and Firestore and simulating an event. - * @param objectName The name of the object (without the bucket). - */ -function mirrorObjectPath(objectName) { - return __awaiter(this, void 0, void 0, function* () { - /** - * Step 1: Skip this if the path isn't valid. - * Step 2: Read the Firestore doc. - * Step 3: Read the GCS Metadata (note this needs to be done after the Firestore read to handle delete timestamps correctly). - * If they don't match, call onObjectChange. - * - If the GCS metadata doesn't exist, simulate a deletion event using the metadata from the Firestore document. - * - * Note that this function is meant to handle dropped events, not necessarily arbirary modifications. - * It is also not going to be able to called on a prefix path. - */ - if (!(0, util_1.shouldMirrorObject)(objectName)) { - return logs.skippedObject(objectName); - } - const paths = (0, util_1.objectNameToFirestorePaths)(objectName); - // Check if the generated Item Document is valid. - if (!(0, util_1.isValidDocumentName)(paths.itemPath)) { - return logs.invalidObjectName(objectName); - } - // Check if every generated Firestore Document will have a valid id. - if (!objectName.split("/").every(util_1.isValidDocumentId)) { - return logs.invalidObjectName(objectName); - } - const existingSnapshot = (yield admin - .firestore() - .doc(paths.itemPath) - .get()); - const currentData = yield getCurrentMetadata(objectName); - if (currentData) { - yield updateFirestore(paths, currentData, false); - } - else if (existingSnapshot.exists) { - var lastEvent = new firestore_1.Timestamp(0, 0); - if (existingSnapshot.data().hasOwnProperty(constants_1.Constants.lastEventField)) { - lastEvent = existingSnapshot.data()[constants_1.Constants.lastEventField]; - } - yield updateFirestore(paths, { lastEvent }, true); - } - else { - logs.skippedMissingPath(objectName); - } - }); -} -exports.mirrorObjectPath = mirrorObjectPath; -/** - * Handler for GCS Object Change events. Will validate the generated Documents before updating Firestore accordingly. - * @param object The GCS Object Metadata object. - * @param eventType The GCS Event type. - */ -function onObjectChange(object, eventType) { - return __awaiter(this, void 0, void 0, function* () { - const isDeletion = (0, util_1.isDeletionEventType)(eventType); - const objectName = object.name; - if (!(0, util_1.shouldMirrorObject)(objectName)) { - return logs.skippedObject(objectName); - } - const paths = (0, util_1.objectNameToFirestorePaths)(objectName); - // Check if the generated Item Document is valid. - if (!(0, util_1.isValidDocumentName)(paths.itemPath)) { - return logs.invalidObjectName(objectName); - } - // Check if every generated Firestore Document will have a valid id. - if (!objectName.split("/").every(util_1.isValidDocumentId)) { - return logs.invalidObjectName(objectName); - } - if (isDeletion) { - // When an object is overwritten, it will fire a deletion/archive event for the original, followed by a finalize - // event for the new one. We try to avoid updating firestore for the first event, so that firestore doesn't - // temporarily show that the document doesn't exist. - const [objectExists] = yield admin - .storage() - .bucket(object.bucket) - .file(objectName) - .exists(); - if (objectExists) { - return logs.skippedOverwrite(objectName, eventType); - } - } - const data = firestoreDocumentData(object, eventType); - yield updateFirestore(paths, data, isDeletion); - }); -} -exports.onObjectChange = onObjectChange; -/** - * Update the Item Document in Firestore with the provided data and perform - * maintenance (creation/deletion) on it's Prefix Documents if necessary. - * @param paths The Item Document path and Prefix Document paths (sorted from root to the parent of the Item Document). - * @param data The data to write to the Item Document (and Prefix Documents if it is a Tombstone). - */ -function updateFirestore(paths, data, isDeletion) { - return __awaiter(this, void 0, void 0, function* () { - const prefixRefs = paths.prefixPaths.map((prefixPath) => admin.firestore().doc(prefixPath)); - const itemRef = admin.firestore().doc(paths.itemPath); - const references = [...prefixRefs, itemRef]; - const timestamp = data[constants_1.Constants.lastEventField]; - let attemptNumber = 0; - return admin - .firestore() - .runTransaction((t) => __awaiter(this, void 0, void 0, function* () { - attemptNumber += 1; - logs.startingTransaction(itemRef.path, attemptNumber); - // Can only write Documents after all reads have been done in the Transaction. - const docsToWrite = []; - const docsToDelete = []; - const transactionReads = []; - // Read the Item Document and it's Tombstone under the Transaction. - const itemTombstoneRef = admin - .firestore() - .doc((0, util_1.mirrorDocumentPathToTombstonePath)(itemRef.path)); - const [itemSnapshot, itemTombstoneSnapshot] = (yield Promise.all([ - t.get(itemRef), - t.get(itemTombstoneRef), - ])); - transactionReads.push(itemSnapshot.ref.path, itemTombstoneSnapshot.ref.path); - if (isStaleEvent(itemSnapshot, data, isDeletion) || - isStaleEvent(itemTombstoneSnapshot, data, isDeletion)) { - // Skip if the event is older than what is in Firestore. - return; - } - if (!isDeletion) { - // Write the Item Document for create/update events - docsToWrite.push({ ref: itemRef, data: data }); - if (itemTombstoneSnapshot.exists) { - docsToDelete.push({ ref: itemTombstoneSnapshot.ref }); - } - } - else { - // Tombstone the Item Document for delete/archive events - docsToWrite.push({ - ref: itemTombstoneSnapshot.ref, - data: data, - }); - if (itemSnapshot.exists) { - docsToDelete.push({ ref: itemRef }); - } - } - // Move up the Prefixes from deepest to shallowest until the root. - // Read each Document and queue up the Documents that we need to modify. - for (let i = references.length - 2; i >= 0; i--) { - const prefixRef = references[i]; - const prefixTombstoneRef = admin - .firestore() - .doc((0, util_1.mirrorDocumentPathToTombstonePath)(prefixRef.path)); - // Read the Prefix Document and it's Tombstone under the Transaction. - const [prefixSnapshot, prefixTombstoneSnapshot] = (yield Promise.all([ - t.get(prefixRef), - t.get(prefixTombstoneRef), - ])); - transactionReads.push(prefixSnapshot.ref.path, prefixTombstoneSnapshot.ref.path); - const child = references[i + 1]; - // Prefix Maintenance, create any Prefix Documents that need to exist, delete - // any that no longer have a child underneath it. - // A reference to an "existing" (non-Tombstone) child is kept on each Prefix - // Document, this is used to skip a Firestore query to determine whether the - // Prefix Document should be pruned when one of it's children is deleted or archived. - if (!isDeletion) { - // Prefix Maintenance for create/update events. - if (prefixTombstoneSnapshot.exists) { - docsToDelete.push({ ref: prefixTombstoneSnapshot.ref }); - } - if (prefixSnapshot.exists) { - // Because we create Prefix Documents from the deepest path to shallowest, once we find a Document - // that has already been created we are guaranteed all remaining Prefix Documents also exist. - break; - } - docsToWrite.push({ - ref: prefixRef, - data: { - [constants_1.Constants.childRefField]: (0, util_1.pathHash)(child.path), - [constants_1.Constants.lastEventField]: timestamp, - }, - }); - } - else { - // Prefix Maintenance for delete/archive events. - // The Prefix Document has been Tombstoned or never existed in the first place. - if (!prefixSnapshot.exists) - break; - // Treat the Prefix Document as one requiring deletion if it is not what we're expecting/invalid. - else if (!isValidPrefixDocument(prefixSnapshot)) { - logs.invalidPrefixDocument(prefixSnapshot.id); - } - // If it already is a Tombstone we can stop. - else if (prefixTombstoneSnapshot.exists) - break; - // This Document doesn't point to a child being deleted, can stop checking parents. - else if (prefixSnapshot.data()[constants_1.Constants.childRefField] !== (0, util_1.pathHash)(child.path)) { - break; - } - // Reference points to Document being deleted, check if the Prefix Document should - // still exist. If it should still exist, update it's child reference. - const items = prefixRef.collection(config_1.default.itemsSubcollectionName); - const prefixes = prefixRef.collection(config_1.default.prefixesSubcollectionName); - const subcollections = [prefixes, items]; - const childDocs = []; - // Try to find "existing" (non-Tombstone) child Documents to replace the child reference. - // A Prefix Document is preferred for the new child reference because they are less likely to be deleted. - for (let i = 0; i < subcollections.length; i++) { - const subcollection = subcollections[i]; - const query = (yield subcollection - .limit(constants_1.Constants.queryLimit - childDocs.length) - .get()); - // New child reference cannot be the old one because its being deleted. - const results = query.docs.filter((doc) => doc.ref.path !== child.path); - childDocs.push(...results); - } - if (childDocs.length === 0) { - // No existing child Documents found. Add a tombstone and continue up. - // Re-using the same Tombstone object that is used to replace Item Documents. - docsToWrite.push({ - ref: prefixTombstoneSnapshot.ref, - data: data, - }); - if (prefixSnapshot.exists) { - docsToDelete.push({ ref: prefixSnapshot.ref }); - } - continue; - } - let newChildPath = null; - // Find a child reference that hasn't been deleted since we ran the query and read under the transaction. - // Gets are transactional whereas the Query performed earlier was not, prevents concurrent transactions - // from deleting the document we've chosen as our new child reference before this transaction is done. - for (let i = 0; i < childDocs.length; i++) { - const newChild = childDocs[i]; - const childSnapshot = (yield t.get(admin.firestore().doc(newChild.ref.path))); - transactionReads.push(childSnapshot.ref.path); - if (childSnapshot.exists) { - newChildPath = childSnapshot.ref.path; - break; - } - } - if (newChildPath === null) { - throw "All Query results have since been Tombstoned. Retrying transaction..."; - } - else { - // Update reference to the new child prefix/object under it. - docsToWrite.push({ - ref: prefixRef, - data: { - [constants_1.Constants.childRefField]: (0, util_1.pathHash)(newChildPath), - [constants_1.Constants.lastEventField]: timestamp, - }, - }); - break; - } - } - } - logs.transactionReads(transactionReads); - // Finished transaction reads. Attempt to write to each queued up document. - logs.transactionWriteAttempt(docsToWrite.length, docsToWrite.map((d) => d.ref.path)); - docsToWrite.forEach((doc) => { - t.set(doc.ref, doc.data); - }); - logs.transactionDeleteAttempt(docsToDelete.length, docsToDelete.map((d) => d.ref.path)); - docsToDelete.forEach((doc) => { - t.delete(doc.ref); - }); - }), { - maxAttempts: constants_1.Constants.transactionAttempts, - }) - .catch((reason) => { - logs.transactionFailure(itemRef.path, reason); - throw reason; - }); - }); -} -/** - * Returns whether an GCS event is stale based on what is currently in Firestore. - * @param existingSnapshot Current Firestore snapshot. - * @param eventData Data to write for the GCS event. - */ -function isStaleEvent(existingSnapshot, eventData, isDeletion) { - // Timestamp on the incoming Firestore document. - const newDocumentTime = eventData[constants_1.Constants.lastEventField]; - if (existingSnapshot.exists) { - // Treat the Document as non-existent if it doesn't have a last-event field. - if (!existingSnapshot.data().hasOwnProperty(constants_1.Constants.lastEventField)) { - logs.missingLastEventField(existingSnapshot.id); - return false; - } - const firestoreTime = (existingSnapshot.data()[constants_1.Constants.lastEventField]); - if (firestoreTime.valueOf() > newDocumentTime.valueOf()) { - logs.abortingStaleEvent(existingSnapshot.ref.path, newDocumentTime, firestoreTime); - // This event is older than what is in Firestore. - return true; - } - else if (firestoreTime.isEqual(newDocumentTime) && !isDeletion) { - logs.staleTieBreak(existingSnapshot.ref.path, newDocumentTime, firestoreTime); - // Break ties in favor of deletion event taking precedence. - return true; - } - logs.eventNotStale(existingSnapshot.ref.path, newDocumentTime, firestoreTime); - return false; - } - logs.docDoesNotExist(existingSnapshot.ref.path); - return false; -} -exports.isStaleEvent = isStaleEvent; -/** - * Construct the Firestore Document fields for the Prefixes Documents & Item Document. - * @param metadata GCS metadata for the relevant object. - * @param eventType The event type. - */ -function firestoreDocumentData(metadata, eventType) { - const timestamp = firestore_1.Timestamp.fromDate(new Date(metadata.updated)); - if ((0, util_1.isDeletionEventType)(eventType)) { - // Instead of deleting Documents when the corresponding Object is deleted, replace it - // with a "Tombstone" to deal with out-of-order function execution. - return { - [constants_1.Constants.lastEventField]: timestamp, - }; - } - const documentData = { - [constants_1.Constants.lastEventField]: timestamp, - [constants_1.Constants.gcsMetadataField]: {}, - }; - const fields = (0, util_1.filterObjectFields)(metadata, config_1.default.metadataFieldFilter); - fields.forEach(({ key, value }) => { - let fieldValue; - if (key === constants_1.Constants.objectCustomMetadataField) { - // Set custom metadata fields - fieldValue = {}; - (0, util_1.filterObjectFields)(value, config_1.default.customMetadataFieldFilter).forEach(({ key, value }) => { - fieldValue[key] = value; - }); - } - else if (constants_1.Constants.numberFields.includes(key)) { - fieldValue = parseInt(value); - } - else if (constants_1.Constants.dateFields.includes(key)) { - fieldValue = new Date(value); - } - else { - fieldValue = value; - } - documentData[constants_1.Constants.gcsMetadataField][key] = fieldValue; - }); - return documentData; -} -/** - * Returns whether a Prefix Document in Firestore has all the valid extension fields. - * @param snapshot The Firestore snapshot. - */ -function isValidPrefixDocument(snapshot) { - const fields = [constants_1.Constants.lastEventField, constants_1.Constants.childRefField]; - for (let i = 0; i < fields.length; i++) { - const field = fields[i]; - if (!snapshot.data().hasOwnProperty(field)) - return false; - } - return true; -} diff --git a/storage-mirror-firestore/functions/lib/util.js b/storage-mirror-firestore/functions/lib/util.js deleted file mode 100644 index 9b511b93..00000000 --- a/storage-mirror-firestore/functions/lib/util.js +++ /dev/null @@ -1,128 +0,0 @@ -"use strict"; -Object.defineProperty(exports, "__esModule", { value: true }); -exports.pathHash = exports.mirrorDocumentPathToTombstonePath = exports.objectNameToFirestorePaths = exports.shouldMirrorObject = exports.isValidDocumentId = exports.isValidDocumentName = exports.isDeletionEventType = exports.filterObjectFields = void 0; -const config_1 = require("./config"); -const crypto = require("crypto"); -/** - * Returns all the key-value pairs in an Object that match the given filter RegEx. - * @param o The Object to filter. - * @param filter The RegEx expression to match with. - */ -function filterObjectFields(o, filter) { - const ret = []; - Object.entries(o).forEach((e) => { - const key = e[0]; - const value = e[1]; - if (key.match(filter) !== null) { - ret.push({ key, value }); - } - }); - return ret; -} -exports.filterObjectFields = filterObjectFields; -/** - * Returns whether the event is an Object Archiving or Object Deletion. - * @param eventType The event type. - */ -function isDeletionEventType(eventType) { - return (eventType === "google.storage.object.delete" || - eventType === "google.storage.object.archive"); -} -exports.isDeletionEventType = isDeletionEventType; -/** - * Returns whether an a Document name (includes Prefixes) is valid in Firestore. - * https://firebase.google.com/docs/firestore/quotas#collections_documents_and_fields - * @param name The name to validate. e.g. `gcs/foo/bar.jpg` - */ -function isValidDocumentName(name) { - // Cannot be more than 100 subcollections deep. - if (name.split("/").length > 100) - return false; - // Cannot be larger than 6 KiB. - if (new Buffer(name).byteLength > 6144) - return false; - return true; -} -exports.isValidDocumentName = isValidDocumentName; -/** - * Returns whether a id is valid for a Document in Firestore. - * https://firebase.google.com/docs/firestore/quotas#collections_documents_and_fields - * @param id The id to validate. e.g. `image.jpg` - */ -function isValidDocumentId(id) { - // Cannot be empty. - if (id.length === 0) - return false; - // Must be no longer than 1500 bytes. - if (new Buffer(id).byteLength > 1500) - return false; - // Cannot solely consist of a single or double period. - if (id === "." || id === "..") - return false; - // Cannot match this regex. - if (id.match(new RegExp("__.*__"))) - return false; - return true; -} -exports.isValidDocumentId = isValidDocumentId; -/** - * Returns whether an Object should be mirrored (matches the configured RegEx). - * @param name The name of the Object. - */ -function shouldMirrorObject(name) { - return name.match(config_1.default.objectNameFilter) !== null; -} -exports.shouldMirrorObject = shouldMirrorObject; -/** - * Returns the Firestore Paths to all the Prefix Documents and the Item Document - * for a given GCS Object name. `prefixPaths` is sorted from the root to the parent - * prefix just before the item path. - * @param name The name of the Object. - */ -function objectNameToFirestorePaths(name) { - const parts = name.split("/"); - const fileName = parts[parts.length - 1]; - // Build array of prefixes for the Object. - let prefix = `${config_1.default.firestoreRoot}`; - const prefixes = []; - for (let i = 0; i < parts.length - 1; i++) { - const part = parts[i]; - prefix += `/${config_1.default.prefixesSubcollectionName}/${part}`; - prefixes.push(prefix); - } - return { - prefixPaths: prefixes, - itemPath: `${prefix}/${config_1.default.itemsSubcollectionName}/${fileName}`, - }; -} -exports.objectNameToFirestorePaths = objectNameToFirestorePaths; -/** - * Returns the relevant Tombstone path for any Item Document or Prefix Document path. - * @param path The Item Document or Prefix Document path. This is assumed to be a valid Document path - * that was generated by the extension (through the `objectNameToFirestorePaths` function or similar). - */ -function mirrorDocumentPathToTombstonePath(path) { - const parts = path.split("/"); - if (parts[parts.length - 2] === config_1.default.itemsSubcollectionName) { - // Tombstone path for an Item Document. - parts[parts.length - 2] = config_1.default.itemsTombstoneSubcollectionName; - } - else if (parts[parts.length - 2] === config_1.default.prefixesSubcollectionName) { - // Tombstone path for an Prefix Document. - parts[parts.length - 2] = config_1.default.prefixesTombstoneSubcollectionName; - } - else { - throw "Invalid Mirror Document Path."; - } - return parts.join("/"); -} -exports.mirrorDocumentPathToTombstonePath = mirrorDocumentPathToTombstonePath; -/** - * Returns a hash of the Firestore Document path. - * @param path The Document path - */ -function pathHash(path) { - const hash = crypto.createHash("md5").update(path).digest("hex"); - return hash; -} -exports.pathHash = pathHash;