From d3b2604f34cfe732708c7da93bf9309569a5495c Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Wed, 8 Oct 2025 10:01:20 -0400 Subject: [PATCH 1/3] HybridContainer --- .../packages/gitrest-base/package.json | 16 +- .../utils/wholeSummary/writeWholeSummary.ts | 5 +- server/gitrest/packages/gitrest/package.json | 4 +- server/gitrest/pnpm-lock.yaml | 128 +++--- .../packages/historian-base/package.json | 16 +- .../packages/historian-base/src/app.ts | 9 +- .../historian-base/src/customizations.ts | 3 +- .../packages/historian-base/src/index.ts | 2 + .../historian-base/src/routes/index.ts | 10 +- .../historian-base/src/routes/summaries.ts | 17 +- .../historian-base/src/routes/utils.ts | 30 +- .../packages/historian-base/src/runner.ts | 9 +- .../historian-base/src/runnerFactory.ts | 5 + .../src/services/definitions.ts | 36 +- .../historian-base/src/services/index.ts | 2 + .../historian/packages/historian/package.json | 4 +- server/historian/pnpm-lock.yaml | 199 ++++----- .../packages/lambdas/src/index.ts | 2 + .../packages/lambdas/src/nexus/connect.ts | 3 +- .../packages/lambdas/src/nexus/index.ts | 4 +- .../packages/lambdas/src/nexus/trace.ts | 29 -- .../packages/lambdas/src/scribe/index.ts | 3 +- .../packages/lambdas/src/scribe/interfaces.ts | 22 +- .../lambdas/src/scribe/lambdaFactory.ts | 6 +- .../lambdas/src/scribe/summaryWriter.ts | 56 ++- .../lambdas/src/test/scribe/lambda.spec.ts | 2 + .../routerlicious/src/scribe/index.ts | 10 +- .../api-report/server-services-client.api.md | 45 ++- .../packages/services-client/src/index.ts | 3 + .../packages/services-client/src/storage.ts | 199 +++++++++ .../src/summaryTreeUploadManager.ts | 1 + .../src/wholeSummaryUploadManager.ts | 4 + .../packages/services-core/src/document.ts | 13 + .../packages/services-core/src/index.ts | 2 + .../packages/services-core/src/tenant.ts | 6 +- .../packages/services-core/src/trace.ts | 34 ++ .../services-shared/src/restLessServer.ts | 2 +- .../services-shared/src/socketIoServer.ts | 4 +- .../packages/services-shared/src/storage.ts | 381 ++++++++++++++---- .../packages/services-telemetry/package.json | 6 + .../src/lumberEventNames.ts | 1 + .../services-telemetry/src/resources.ts | 1 + ...rverServicesTelemetryPrevious.generated.ts | 2 + .../services-utils/src/asyncContext.ts | 4 +- .../packages/services/src/documentManager.ts | 28 +- .../packages/services/src/tenant.ts | 130 +++++- 46 files changed, 1138 insertions(+), 360 deletions(-) create mode 100644 server/routerlicious/packages/services-core/src/trace.ts diff --git a/server/gitrest/packages/gitrest-base/package.json b/server/gitrest/packages/gitrest-base/package.json index 5f7930f28da5..3cba3654d04a 100644 --- a/server/gitrest/packages/gitrest-base/package.json +++ b/server/gitrest/packages/gitrest-base/package.json @@ -52,15 +52,15 @@ }, "dependencies": { "@fluidframework/common-utils": "^1.1.1", - "@fluidframework/gitresources": "6.0.0-340759", - "@fluidframework/protocol-base": "6.0.0-340759", + "@fluidframework/gitresources": "8.0.0-347501", + "@fluidframework/protocol-base": "8.0.0-347501", "@fluidframework/protocol-definitions": "^3.2.0", - "@fluidframework/server-services-client": "6.0.0-340759", - "@fluidframework/server-services-core": "6.0.0-340759", - "@fluidframework/server-services-shared": "6.0.0-340759", - "@fluidframework/server-services-telemetry": "6.0.0-340759", - "@fluidframework/server-services-utils": "6.0.0-340759", - "@fluidframework/server-test-utils": "6.0.0-340759", + "@fluidframework/server-services-client": "8.0.0-347501", + "@fluidframework/server-services-core": "8.0.0-347501", + "@fluidframework/server-services-shared": "8.0.0-347501", + "@fluidframework/server-services-telemetry": "8.0.0-347501", + "@fluidframework/server-services-utils": "8.0.0-347501", + "@fluidframework/server-test-utils": "8.0.0-347501", "async-mutex": "^0.3.2", "axios": "^1.8.4", "body-parser": "^1.20.3", diff --git a/server/gitrest/packages/gitrest-base/src/utils/wholeSummary/writeWholeSummary.ts b/server/gitrest/packages/gitrest-base/src/utils/wholeSummary/writeWholeSummary.ts index f33b8e5e4bac..b006ec275ae4 100644 --- a/server/gitrest/packages/gitrest-base/src/utils/wholeSummary/writeWholeSummary.ts +++ b/server/gitrest/packages/gitrest-base/src/utils/wholeSummary/writeWholeSummary.ts @@ -195,6 +195,7 @@ async function createNewSummaryVersion( isNewDocument: boolean, sequenceNumber: number, options: IWholeSummaryOptions, + commitDateStr?: string, ): Promise { const commitMessage = isNewDocument ? "New document" @@ -203,7 +204,7 @@ async function createNewSummaryVersion( `Summary @${sequenceNumber}`; const commitParams: ICreateCommitParams = { author: { - date: new Date().toISOString(), + date: commitDateStr ?? new Date().toISOString(), email: "dummy@microsoft.com", name: "GitRest Service", }, @@ -215,6 +216,7 @@ async function createNewSummaryVersion( GitRestLumberEventName.CreateSummaryVersion, options.lumberjackProperties, ); + try { const commit = await options.repoManager.createCommit(commitParams); writeSummaryVersionMetric.success("Successfully created summary version as Git commit."); @@ -316,6 +318,7 @@ export async function writeContainerSummary( isNewDocument, payload.sequenceNumber, options, + payload.summaryTime, ); // Create or update the document ref to reference the new commit. diff --git a/server/gitrest/packages/gitrest/package.json b/server/gitrest/packages/gitrest/package.json index f38520c8aafc..983b73f75a09 100644 --- a/server/gitrest/packages/gitrest/package.json +++ b/server/gitrest/packages/gitrest/package.json @@ -28,8 +28,8 @@ }, "dependencies": { "@fluidframework/gitrest-base": "workspace:~", - "@fluidframework/server-services-shared": "6.0.0-340759", - "@fluidframework/server-services-utils": "6.0.0-340759", + "@fluidframework/server-services-shared": "8.0.0-347501", + "@fluidframework/server-services-utils": "8.0.0-347501", "body-parser": "^1.20.3", "compression": "^1.7.3", "cors": "^2.8.5", diff --git a/server/gitrest/pnpm-lock.yaml b/server/gitrest/pnpm-lock.yaml index b9d2d8c24fa6..dcfe35d017b1 100644 --- a/server/gitrest/pnpm-lock.yaml +++ b/server/gitrest/pnpm-lock.yaml @@ -99,11 +99,11 @@ importers: specifier: workspace:~ version: link:../gitrest-base '@fluidframework/server-services-shared': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-utils': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 body-parser: specifier: ^1.20.3 version: 1.20.3 @@ -199,32 +199,32 @@ importers: specifier: ^1.1.1 version: 1.1.1 '@fluidframework/gitresources': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/protocol-base': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/protocol-definitions': specifier: ^3.2.0 version: 3.2.0 '@fluidframework/server-services-client': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-core': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-shared': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-telemetry': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-utils': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-test-utils': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 async-mutex: specifier: ^0.3.2 version: 0.3.2 @@ -647,32 +647,32 @@ packages: '@fluidframework/eslint-config-fluid@6.1.0': resolution: {integrity: sha512-jPVt7ylQsCyptYmhNPby57qOuNGzvjteDSy4/x+hKzibgJ76yqRAnTVMjXAeR9dlLUfiXydhOcecNvx4Zt3IGQ==} - '@fluidframework/gitresources@6.0.0-340759': - resolution: {integrity: sha512-G3XrO17K1PEvKd/dMqLUExQVzokyfkC78412DiuSwembp6zn+/jKU9qETeSTFYiSPQCVcVS6Sf3cIw9ifUri3w==} + '@fluidframework/gitresources@8.0.0-347501': + resolution: {integrity: sha512-+6av8aQJUT2hQjfnp079eZYBc7hva53BSywMuTNd57dMi1ThPOII5jc7nAFJOP955jo5o8hqSmwasI76tS1DXg==} - '@fluidframework/protocol-base@6.0.0-340759': - resolution: {integrity: sha512-crUqTz46GQovlfpetUjDxO/Jv+awLYjfVqbWdIZT8EifW8xobZovld7m0+mYh62mmmHZD3KVzf6ROUL2CXI0lw==} + '@fluidframework/protocol-base@8.0.0-347501': + resolution: {integrity: sha512-VqGjB1Zi0sOEvkRXve3KXCtIuy1D50V3bGO4+k/pGWb6CzCtawJEH6yNidoJrjINZ1fc1WAlZGhoegXsoHHQZg==} '@fluidframework/protocol-definitions@3.2.0': resolution: {integrity: sha512-xgcyMN4uF6dAp2/XYFSHvGFITIV7JbVt3itA+T0c71/lZjq/HU/a/ClPIxfl9AEN0RbtuR/1n5LP4FXSV9j0hA==} - '@fluidframework/server-services-client@6.0.0-340759': - resolution: {integrity: sha512-dfDciBNAZstvJ2mzzq9gXGCdrpZclziYQC/k0+P3pMmxXgEjpOnsfXPJGeln5kIPZTyQghOgbQvDQiYPpx98Qw==} + '@fluidframework/server-services-client@8.0.0-347501': + resolution: {integrity: sha512-m1vMEnq/hGs1Qj/f22GifbQSqTFhTtz4vmn+wSOGOu2/9cJQoNbDoRXxaUTUW/DNw7g6LGH+sUs38OZ980Uvkg==} - '@fluidframework/server-services-core@6.0.0-340759': - resolution: {integrity: sha512-AYdECEsMhDwnkq0HjUR2w2DdvKEvSNb7O7hhCFO55JN5Q/bKHtvOGai92VGN2r1p4Kc/fLqPEoI3b1g6nmSn4A==} + '@fluidframework/server-services-core@8.0.0-347501': + resolution: {integrity: sha512-tIpNNiHwDJM4OE2nVSxaZFq1ZJUHd6ojMAFrI0sZKzH4hV9eKgmXzaqoZuJ+sJjcXwJMphQezRjlnwDWMzaFxg==} - '@fluidframework/server-services-shared@6.0.0-340759': - resolution: {integrity: sha512-MmlhjWXfehWqCPQ2ce639c0VpDuYdZEyUs25csNJFo8EwGBnXS6DqWDaiEDgN1aCz0kTk0eCAeLiGSqWSjR0Ng==} + '@fluidframework/server-services-shared@8.0.0-347501': + resolution: {integrity: sha512-UriHDJla6DfT435n+VPnJm8xxUZFVyTSN9QYE7XpeVtXX0FC8NvjVJ1fN/Ig99C2TwYKXjxAwbf0EgB3PWtJxA==} - '@fluidframework/server-services-telemetry@6.0.0-340759': - resolution: {integrity: sha512-I0NbzN2UTXQxI0Dh5hPuTUUKq84sSY9g5w1wq51jdQXu/Kc0MGzg4E+2wXzOXrkxAP32CB+7sMmIeNCUvuiCDg==} + '@fluidframework/server-services-telemetry@8.0.0-347501': + resolution: {integrity: sha512-+aCQGcyMl9Db+nvxT+B5Kjma8+LC52xbOEiz7+qkqSYEuF79Ncpjybe4Lsy98LjsAqtidf7YUJ+JLR5IZP7E/Q==} - '@fluidframework/server-services-utils@6.0.0-340759': - resolution: {integrity: sha512-4FdaT389HTKFdQxstgIni1CZPI6rDtQQTfhX959QkjQIeKmU62xhJuSCEryKNFOlP3xQZgEodwgnXT2gHmy5Sw==} + '@fluidframework/server-services-utils@8.0.0-347501': + resolution: {integrity: sha512-8g04jsKixcluWoT1ucVKLUsp1tcuknFqd7rkl+PhZjSnusUopN8sEBFwwbZ/RVuGZa/K13X9o6Tph0KeUDLyLA==} - '@fluidframework/server-test-utils@6.0.0-340759': - resolution: {integrity: sha512-mPPleCzwqz1DZFvf8Zk3xroQZGfzm2fqRsajXFymxO7b/ea8IE1l3i16ZB0m9jrmctq/PFzAGSosarJB1XdgxQ==} + '@fluidframework/server-test-utils@8.0.0-347501': + resolution: {integrity: sha512-vtd19R6BhicC8dPCt6zvjM6dPfu1DIOyv/gJty2l7Rf/Kjya/iIn4ibB4jTm1IaJ5DjGxp58qC6DUnAUHMJmcw==} '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} @@ -6779,22 +6779,22 @@ snapshots: - supports-color - typescript - '@fluidframework/gitresources@6.0.0-340759': {} + '@fluidframework/gitresources@8.0.0-347501': {} - '@fluidframework/protocol-base@6.0.0-340759': + '@fluidframework/protocol-base@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 events_pkg: events@3.3.0 '@fluidframework/protocol-definitions@3.2.0': {} - '@fluidframework/server-services-client@6.0.0-340759': + '@fluidframework/server-services-client@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 - '@fluidframework/protocol-base': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 + '@fluidframework/protocol-base': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 axios: 1.8.4(debug@4.4.3) crc-32: 1.2.0 @@ -6807,13 +6807,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-services-core@6.0.0-340759': + '@fluidframework/server-services-core@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 '@types/nconf': 0.10.3 '@types/node': 18.17.7 debug: 4.4.3(supports-color@8.1.1) @@ -6822,16 +6822,16 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-services-shared@6.0.0-340759': + '@fluidframework/server-services-shared@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 - '@fluidframework/protocol-base': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 + '@fluidframework/protocol-base': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 - '@fluidframework/server-services-utils': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 + '@fluidframework/server-services-utils': 8.0.0-347501 '@socket.io/redis-adapter': 8.3.0(socket.io-adapter@2.5.5) '@socket.io/sticky': 1.0.4 body-parser: 1.20.3 @@ -6854,7 +6854,7 @@ snapshots: - supports-color - utf-8-validate - '@fluidframework/server-services-telemetry@6.0.0-340759': + '@fluidframework/server-services-telemetry@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 json-stringify-safe: 5.0.1 @@ -6862,12 +6862,12 @@ snapshots: serialize-error: 8.1.0 uuid: 11.1.0 - '@fluidframework/server-services-utils@6.0.0-340759': + '@fluidframework/server-services-utils@8.0.0-347501': dependencies: '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 debug: 4.4.3(supports-color@8.1.1) express: 4.21.2 ioredis: 5.6.1 @@ -6884,15 +6884,15 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-test-utils@6.0.0-340759': + '@fluidframework/server-test-utils@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 - '@fluidframework/protocol-base': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 + '@fluidframework/protocol-base': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 '@types/ioredis-mock': 8.2.6(ioredis@5.6.1) assert: 2.1.0 debug: 4.4.3(supports-color@8.1.1) diff --git a/server/historian/packages/historian-base/package.json b/server/historian/packages/historian-base/package.json index e44d2c1c10e3..3a188c71b81b 100644 --- a/server/historian/packages/historian-base/package.json +++ b/server/historian/packages/historian-base/package.json @@ -53,14 +53,14 @@ }, "dependencies": { "@fluidframework/common-utils": "^1.1.1", - "@fluidframework/gitresources": "6.0.0-340759", + "@fluidframework/gitresources": "8.0.0-347501", "@fluidframework/protocol-definitions": "^3.2.0", - "@fluidframework/server-services": "6.0.0-340759", - "@fluidframework/server-services-client": "6.0.0-340759", - "@fluidframework/server-services-core": "6.0.0-340759", - "@fluidframework/server-services-shared": "6.0.0-340759", - "@fluidframework/server-services-telemetry": "6.0.0-340759", - "@fluidframework/server-services-utils": "6.0.0-340759", + "@fluidframework/server-services": "8.0.0-347501", + "@fluidframework/server-services-client": "8.0.0-347501", + "@fluidframework/server-services-core": "8.0.0-347501", + "@fluidframework/server-services-shared": "8.0.0-347501", + "@fluidframework/server-services-telemetry": "8.0.0-347501", + "@fluidframework/server-services-utils": "8.0.0-347501", "@types/ioredis-mock": "^8.2.6", "axios": "^1.8.4", "body-parser": "^1.20.3", @@ -79,7 +79,7 @@ "devDependencies": { "@fluidframework/build-common": "^2.0.3", "@fluidframework/eslint-config-fluid": "^6.1.0", - "@fluidframework/server-test-utils": "6.0.0-340759", + "@fluidframework/server-test-utils": "8.0.0-347501", "@types/compression": "0.0.36", "@types/cors": "^2.8.4", "@types/debug": "^4.1.5", diff --git a/server/historian/packages/historian-base/src/app.ts b/server/historian/packages/historian-base/src/app.ts index 456c442293b1..3c7943df8efe 100644 --- a/server/historian/packages/historian-base/src/app.ts +++ b/server/historian/packages/historian-base/src/app.ts @@ -34,7 +34,12 @@ import express from "express"; import type * as nconf from "nconf"; import * as routes from "./routes"; -import type { ICache, ITenantService, ISimplifiedCustomDataRetriever } from "./services"; +import type { + ICache, + ITenantService, + ISimplifiedCustomDataRetriever, + IPostEphemeralContainerChecker, +} from "./services"; import { Constants, getDocumentIdFromRequest, getTenantIdFromRequest } from "./utils"; export function create( @@ -51,6 +56,7 @@ export function create( ephemeralDocumentTTLSec?: number, readinessCheck?: IReadinessCheck, simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever, + postEphemeralContainerChecker?: IPostEphemeralContainerChecker, ) { // Express app configuration const app: express.Express = express(); @@ -131,6 +137,7 @@ export function create( denyList, ephemeralDocumentTTLSec, simplifiedCustomDataRetriever, + postEphemeralContainerChecker, ); app.use(apiRoutes.git.blobs); app.use(apiRoutes.git.refs); diff --git a/server/historian/packages/historian-base/src/customizations.ts b/server/historian/packages/historian-base/src/customizations.ts index 340d5c7e96b0..150357ca342e 100644 --- a/server/historian/packages/historian-base/src/customizations.ts +++ b/server/historian/packages/historian-base/src/customizations.ts @@ -10,7 +10,7 @@ import type { } from "@fluidframework/server-services-core"; import type { IRedisClientConnectionManager } from "@fluidframework/server-services-utils"; -import type { ISimplifiedCustomDataRetriever } from "./services"; +import type { IPostEphemeralContainerChecker, ISimplifiedCustomDataRetriever } from "./services"; export interface IHistorianResourcesCustomizations { storageNameRetriever?: IStorageNameRetriever; @@ -20,4 +20,5 @@ export interface IHistorianResourcesCustomizations { redisClientConnectionManagerForInvalidTokenCache?: IRedisClientConnectionManager; readinessCheck?: IReadinessCheck; simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever; + postEphemeralContainerChecker?: IPostEphemeralContainerChecker; } diff --git a/server/historian/packages/historian-base/src/index.ts b/server/historian/packages/historian-base/src/index.ts index 15aee3bef6f2..8e1a7324953a 100644 --- a/server/historian/packages/historian-base/src/index.ts +++ b/server/historian/packages/historian-base/src/index.ts @@ -23,6 +23,8 @@ export { type ITenant, type ITenantCustomDataExternal, type ITenantService, + type IPostEphemeralContainerChecker, + type ICreateGitServiceArgs, RedisCache, RedisTenantCache, RestGitService, diff --git a/server/historian/packages/historian-base/src/routes/index.ts b/server/historian/packages/historian-base/src/routes/index.ts index 5c2f20428e4d..ce7abfebcd02 100644 --- a/server/historian/packages/historian-base/src/routes/index.ts +++ b/server/historian/packages/historian-base/src/routes/index.ts @@ -13,7 +13,12 @@ import type { import type { Router } from "express"; import type * as nconf from "nconf"; -import type { ICache, ITenantService, ISimplifiedCustomDataRetriever } from "../services"; +import type { + ICache, + ITenantService, + ISimplifiedCustomDataRetriever, + IPostEphemeralContainerChecker, +} from "../services"; /* eslint-disable import/no-internal-modules */ import * as blobs from "./git/blobs"; @@ -56,6 +61,7 @@ export function create( denyList?: IDenyList, ephemeralDocumentTTLSec?: number, simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever, + postEphemeralContainerChecker?: IPostEphemeralContainerChecker, ): IRoutes { const commonRouteParams: CommonRouteParams = [ config, @@ -83,6 +89,6 @@ export function create( contents: contents.create(...commonRouteParams), headers: headers.create(...commonRouteParams), }, - summaries: summaries.create(...commonRouteParams), + summaries: summaries.create(...commonRouteParams, postEphemeralContainerChecker), }; } diff --git a/server/historian/packages/historian-base/src/routes/summaries.ts b/server/historian/packages/historian-base/src/routes/summaries.ts index 400c285a6312..e2ebfeb673eb 100644 --- a/server/historian/packages/historian-base/src/routes/summaries.ts +++ b/server/historian/packages/historian-base/src/routes/summaries.ts @@ -24,10 +24,16 @@ import { throttle, } from "@fluidframework/server-services-utils"; import { Router } from "express"; +import type { Query } from "express-serve-static-core"; import type * as nconf from "nconf"; import winston from "winston"; -import type { ICache, ITenantService, ISimplifiedCustomDataRetriever } from "../services"; +import type { + ICache, + ITenantService, + ISimplifiedCustomDataRetriever, + IPostEphemeralContainerChecker, +} from "../services"; import { parseToken, Constants } from "../utils"; import * as utils from "./utils"; @@ -44,6 +50,7 @@ export function create( denyList?: IDenyList, ephemeralDocumentTTLSec?: number, simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever, + postEphemeralContainerChecker?: IPostEphemeralContainerChecker, ): Router { const router: Router = Router(); const ignoreIsEphemeralFlag: boolean = config.get("ignoreEphemeralFlag") ?? true; @@ -98,6 +105,7 @@ export function create( authorization: string | undefined, sha: string, useCache: boolean, + query?: Query, ): Promise { const service = await utils.createGitService({ config, @@ -108,6 +116,8 @@ export function create( documentManager, cache, ephemeralDocumentTTLSec, + postEphemeralContainerChecker, + query, }); return service.getSummary(sha, useCache); } @@ -120,6 +130,7 @@ export function create( storageName?: string, isEphemeralContainer?: boolean, ignoreEphemeralFlag?: boolean, + query?: Query, ): Promise { const service = await utils.createGitService({ config, @@ -134,6 +145,8 @@ export function create( isEphemeralContainer, ephemeralDocumentTTLSec, simplifiedCustomDataRetriever, + postEphemeralContainerChecker, + query, }); return service.createSummary(params, initial); } @@ -178,6 +191,7 @@ export function create( request.get("Authorization"), request.params.sha, useCache, + request.query, ); utils.handleResponse( @@ -243,6 +257,7 @@ export function create( request.get("StorageName"), isEphemeralContainer, ignoreIsEphemeralFlag, + request.query, ); utils.handleResponse(summaryP, response, false, undefined, 201); diff --git a/server/historian/packages/historian-base/src/routes/utils.ts b/server/historian/packages/historian-base/src/routes/utils.ts index 1f6d9c6dc0a8..884c51d910bc 100644 --- a/server/historian/packages/historian-base/src/routes/utils.ts +++ b/server/historian/packages/historian-base/src/routes/utils.ts @@ -30,6 +30,7 @@ import { RestGitService, type ITenantCustomDataExternal, type ISimplifiedCustomDataRetriever, + type ICreateGitServiceArgs, } from "../services"; import { containsPathTraversal, parseToken } from "../utils"; @@ -51,22 +52,6 @@ export type CommonRouteParams = [ simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever, ]; -export interface ICreateGitServiceArgs { - config: nconf.Provider; - tenantId: string; - authorization: string | undefined; - tenantService: ITenantService; - storageNameRetriever?: IStorageNameRetriever; - documentManager: IDocumentManager; - cache?: ICache; - initialUpload?: boolean; - storageName?: string; - allowDisabledTenant?: boolean; - isEphemeralContainer?: boolean; - ephemeralDocumentTTLSec?: number; // 24 hours - simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever; -} - function getEphemeralContainerCacheKey(documentId: string): string { return `isEphemeralContainer:${documentId}`; } @@ -266,6 +251,7 @@ export async function createGitService(createArgs: ICreateGitServiceArgs): Promi isEphemeralContainer, ephemeralDocumentTTLSec, simplifiedCustomDataRetriever, + postEphemeralContainerChecker, } = { ...createArgs }; if (!authorization) { throw new NetworkError(403, "Authorization header is missing."); @@ -303,6 +289,16 @@ export async function createGitService(createArgs: ICreateGitServiceArgs): Promi Lumberjack.info(`Document is ephemeral.`, getLumberBaseProperties(documentId, tenantId)); } + let postIsEphemeral: boolean = isEphemeral; + if (postEphemeralContainerChecker !== undefined) { + postIsEphemeral = await postEphemeralContainerChecker.postEphemeralContainerCheck( + tenantId, + documentId, + isEphemeral, + createArgs, + ); + } + const calculatedStorageName = initialUpload && storageName ? storageName @@ -315,7 +311,7 @@ export async function createGitService(createArgs: ICreateGitServiceArgs): Promi cache, calculatedStorageName, storageUrl, - isEphemeral, + postIsEphemeral, maxCacheableSummarySize, simplifiedCustomData, ); diff --git a/server/historian/packages/historian-base/src/runner.ts b/server/historian/packages/historian-base/src/runner.ts index 635448f6367f..79fb272cb526 100644 --- a/server/historian/packages/historian-base/src/runner.ts +++ b/server/historian/packages/historian-base/src/runner.ts @@ -20,7 +20,12 @@ import type { Provider } from "nconf"; import * as winston from "winston"; import * as app from "./app"; -import type { ICache, ITenantService, ISimplifiedCustomDataRetriever } from "./services"; +import type { + ICache, + ITenantService, + ISimplifiedCustomDataRetriever, + IPostEphemeralContainerChecker, +} from "./services"; export class HistorianRunner implements IRunner { private server: IWebServer | undefined; @@ -42,6 +47,7 @@ export class HistorianRunner implements IRunner { private readonly ephemeralDocumentTTLSec?: number, private readonly readinessCheck?: IReadinessCheck, private readonly simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever, + private readonly PostEphemeralContainerChecker?: IPostEphemeralContainerChecker, ) {} // eslint-disable-next-line @typescript-eslint/promise-function-async @@ -62,6 +68,7 @@ export class HistorianRunner implements IRunner { this.ephemeralDocumentTTLSec, this.readinessCheck, this.simplifiedCustomDataRetriever, + this.PostEphemeralContainerChecker, ); historian.set("port", this.port); diff --git a/server/historian/packages/historian-base/src/runnerFactory.ts b/server/historian/packages/historian-base/src/runnerFactory.ts index 01ad75555133..bc6baf759bb6 100644 --- a/server/historian/packages/historian-base/src/runnerFactory.ts +++ b/server/historian/packages/historian-base/src/runnerFactory.ts @@ -40,6 +40,7 @@ export class HistorianResources implements core.IResources { public readonly ephemeralDocumentTTLSec?: number, public readonly readinessCheck?: core.IReadinessCheck, public readonly simplifiedCustomDataRetriever?: historianServices.ISimplifiedCustomDataRetriever, + public readonly postEphemeralContainerChecker?: historianServices.IPostEphemeralContainerChecker, ) { const httpServerConfig: services.IHttpServerConfig = config.get("system:httpServer"); this.webServerFactory = new services.BasicWebServerFactory(httpServerConfig); @@ -265,6 +266,8 @@ export class HistorianResourcesFactory implements core.IResourcesFactory; } +export interface ICreateGitServiceArgs { + config: nconf.Provider; + tenantId: string; + authorization: string | undefined; + tenantService: ITenantService; + storageNameRetriever?: IStorageNameRetriever; + documentManager: IDocumentManager; + cache?: ICache; + initialUpload?: boolean; + storageName?: string; + allowDisabledTenant?: boolean; + isEphemeralContainer?: boolean; + ephemeralDocumentTTLSec?: number; // 24 hours + simplifiedCustomDataRetriever?: ISimplifiedCustomDataRetriever; + postEphemeralContainerChecker?: IPostEphemeralContainerChecker; + query?: Query; +} + export interface ITenantService { /** * Retrieves the storage provider details for the given tenant. @@ -106,3 +131,12 @@ export interface IOauthAccessInfo { export interface ISimplifiedCustomDataRetriever { get(customData: ITenantCustomData): string; } + +export interface IPostEphemeralContainerChecker { + postEphemeralContainerCheck: ( + tenantId: string, + documentId: string, + isEphemeral: boolean, + createArgs: ICreateGitServiceArgs, + ) => Promise; +} diff --git a/server/historian/packages/historian-base/src/services/index.ts b/server/historian/packages/historian-base/src/services/index.ts index fe7da18485b6..795cc831f42b 100644 --- a/server/historian/packages/historian-base/src/services/index.ts +++ b/server/historian/packages/historian-base/src/services/index.ts @@ -14,6 +14,8 @@ export type { ITenantCustomDataExternal, ITenantService, ISimplifiedCustomDataRetriever, + IPostEphemeralContainerChecker, + ICreateGitServiceArgs, } from "./definitions"; export { RedisCache } from "./redisCache"; export { RedisTenantCache } from "./redisTenantCache"; diff --git a/server/historian/packages/historian/package.json b/server/historian/packages/historian/package.json index 3f02f7ede2b9..083e7bae1890 100644 --- a/server/historian/packages/historian/package.json +++ b/server/historian/packages/historian/package.json @@ -29,8 +29,8 @@ }, "dependencies": { "@fluidframework/historian-base": "workspace:~", - "@fluidframework/server-services-shared": "6.0.0-340759", - "@fluidframework/server-services-utils": "6.0.0-340759", + "@fluidframework/server-services-shared": "8.0.0-347501", + "@fluidframework/server-services-utils": "8.0.0-347501", "body-parser": "^1.20.3", "compression": "^1.7.3", "cors": "^2.8.5", diff --git a/server/historian/pnpm-lock.yaml b/server/historian/pnpm-lock.yaml index ee18743e7ad6..5171ef2d34d5 100644 --- a/server/historian/pnpm-lock.yaml +++ b/server/historian/pnpm-lock.yaml @@ -84,11 +84,11 @@ importers: specifier: workspace:~ version: link:../historian-base '@fluidframework/server-services-shared': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-utils': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 body-parser: specifier: ^1.20.3 version: 1.20.3 @@ -181,29 +181,29 @@ importers: specifier: ^1.1.1 version: 1.1.1 '@fluidframework/gitresources': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/protocol-definitions': specifier: ^3.2.0 version: 3.2.0 '@fluidframework/server-services': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-client': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-core': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-shared': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-telemetry': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@fluidframework/server-services-utils': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@types/ioredis-mock': specifier: ^8.2.6 version: 8.2.6(ioredis@5.6.1) @@ -254,8 +254,8 @@ importers: specifier: ^6.1.0 version: 6.1.0(eslint@8.57.1)(typescript@5.1.6) '@fluidframework/server-test-utils': - specifier: 6.0.0-340759 - version: 6.0.0-340759 + specifier: 8.0.0-347501 + version: 8.0.0-347501 '@types/compression': specifier: 0.0.36 version: 0.0.36 @@ -575,44 +575,44 @@ packages: '@fluidframework/eslint-config-fluid@6.1.0': resolution: {integrity: sha512-jPVt7ylQsCyptYmhNPby57qOuNGzvjteDSy4/x+hKzibgJ76yqRAnTVMjXAeR9dlLUfiXydhOcecNvx4Zt3IGQ==} - '@fluidframework/gitresources@6.0.0-340759': - resolution: {integrity: sha512-G3XrO17K1PEvKd/dMqLUExQVzokyfkC78412DiuSwembp6zn+/jKU9qETeSTFYiSPQCVcVS6Sf3cIw9ifUri3w==} + '@fluidframework/gitresources@8.0.0-347501': + resolution: {integrity: sha512-+6av8aQJUT2hQjfnp079eZYBc7hva53BSywMuTNd57dMi1ThPOII5jc7nAFJOP955jo5o8hqSmwasI76tS1DXg==} - '@fluidframework/protocol-base@6.0.0-340759': - resolution: {integrity: sha512-crUqTz46GQovlfpetUjDxO/Jv+awLYjfVqbWdIZT8EifW8xobZovld7m0+mYh62mmmHZD3KVzf6ROUL2CXI0lw==} + '@fluidframework/protocol-base@8.0.0-347501': + resolution: {integrity: sha512-VqGjB1Zi0sOEvkRXve3KXCtIuy1D50V3bGO4+k/pGWb6CzCtawJEH6yNidoJrjINZ1fc1WAlZGhoegXsoHHQZg==} '@fluidframework/protocol-definitions@3.2.0': resolution: {integrity: sha512-xgcyMN4uF6dAp2/XYFSHvGFITIV7JbVt3itA+T0c71/lZjq/HU/a/ClPIxfl9AEN0RbtuR/1n5LP4FXSV9j0hA==} - '@fluidframework/server-services-client@6.0.0-340759': - resolution: {integrity: sha512-dfDciBNAZstvJ2mzzq9gXGCdrpZclziYQC/k0+P3pMmxXgEjpOnsfXPJGeln5kIPZTyQghOgbQvDQiYPpx98Qw==} + '@fluidframework/server-services-client@8.0.0-347501': + resolution: {integrity: sha512-m1vMEnq/hGs1Qj/f22GifbQSqTFhTtz4vmn+wSOGOu2/9cJQoNbDoRXxaUTUW/DNw7g6LGH+sUs38OZ980Uvkg==} - '@fluidframework/server-services-core@6.0.0-340759': - resolution: {integrity: sha512-AYdECEsMhDwnkq0HjUR2w2DdvKEvSNb7O7hhCFO55JN5Q/bKHtvOGai92VGN2r1p4Kc/fLqPEoI3b1g6nmSn4A==} + '@fluidframework/server-services-core@8.0.0-347501': + resolution: {integrity: sha512-tIpNNiHwDJM4OE2nVSxaZFq1ZJUHd6ojMAFrI0sZKzH4hV9eKgmXzaqoZuJ+sJjcXwJMphQezRjlnwDWMzaFxg==} - '@fluidframework/server-services-ordering-kafkanode@6.0.0-340759': - resolution: {integrity: sha512-j3gAOx0JcPKV3sij7GP/x/Q0M7cj98C+HtXtoVI/Q12qs1G5REkK0+6O2VRAHho/VgruVlfL15LJ/PgzlOUh8g==} + '@fluidframework/server-services-ordering-kafkanode@8.0.0-347501': + resolution: {integrity: sha512-+/uwufqnun2s5z+qnFZskox+GRmGYWWhUElJHbBnJ1gk+EZiMQLBVxmRNffSb6o51JmsCc7Ybllb3fF2wAKAdg==} - '@fluidframework/server-services-ordering-rdkafka@6.0.0-340759': - resolution: {integrity: sha512-ahtFGPFwLkCM7pv4CRBQ6EenGHMCa92gIvn3jHxfuxyvPRQrL2ulFSimaciYN7m0wQ5vhR1ADVHLFjMUtVBEVA==} + '@fluidframework/server-services-ordering-rdkafka@8.0.0-347501': + resolution: {integrity: sha512-IKLp2drE/Y4WrfIoSq9sWXX4UbdnnbqjWIPR5K990bMdQ//C8pwCE14KLYJCxhmyHH1rLy6DkGJ6rS2c1oSvaw==} - '@fluidframework/server-services-ordering-zookeeper@6.0.0-340759': - resolution: {integrity: sha512-3VJZKUZIUcVC0SimKNqCE3bfAKe0s7fa38uho9ytddtscHGWMBzhaaTHea0W57ISH2OBu6SasnWeByLe8syKaw==} + '@fluidframework/server-services-ordering-zookeeper@8.0.0-347501': + resolution: {integrity: sha512-cBjC2SWPY6ZpFeAxKNuh5YT6c1ZJGvPsKVxmQFuOkB1hsFLKH6aINjx+BD4BM6uw6HEipgTutsVmXUH++3Mz8g==} - '@fluidframework/server-services-shared@6.0.0-340759': - resolution: {integrity: sha512-MmlhjWXfehWqCPQ2ce639c0VpDuYdZEyUs25csNJFo8EwGBnXS6DqWDaiEDgN1aCz0kTk0eCAeLiGSqWSjR0Ng==} + '@fluidframework/server-services-shared@8.0.0-347501': + resolution: {integrity: sha512-UriHDJla6DfT435n+VPnJm8xxUZFVyTSN9QYE7XpeVtXX0FC8NvjVJ1fN/Ig99C2TwYKXjxAwbf0EgB3PWtJxA==} - '@fluidframework/server-services-telemetry@6.0.0-340759': - resolution: {integrity: sha512-I0NbzN2UTXQxI0Dh5hPuTUUKq84sSY9g5w1wq51jdQXu/Kc0MGzg4E+2wXzOXrkxAP32CB+7sMmIeNCUvuiCDg==} + '@fluidframework/server-services-telemetry@8.0.0-347501': + resolution: {integrity: sha512-+aCQGcyMl9Db+nvxT+B5Kjma8+LC52xbOEiz7+qkqSYEuF79Ncpjybe4Lsy98LjsAqtidf7YUJ+JLR5IZP7E/Q==} - '@fluidframework/server-services-utils@6.0.0-340759': - resolution: {integrity: sha512-4FdaT389HTKFdQxstgIni1CZPI6rDtQQTfhX959QkjQIeKmU62xhJuSCEryKNFOlP3xQZgEodwgnXT2gHmy5Sw==} + '@fluidframework/server-services-utils@8.0.0-347501': + resolution: {integrity: sha512-8g04jsKixcluWoT1ucVKLUsp1tcuknFqd7rkl+PhZjSnusUopN8sEBFwwbZ/RVuGZa/K13X9o6Tph0KeUDLyLA==} - '@fluidframework/server-services@6.0.0-340759': - resolution: {integrity: sha512-ZUjm7aoU4X0YLrklh3VGQSHJPFKmeBvcufT+CBMkwnfnO7kwaTp30guRbdyLxzpq1yff8ZDgS9unTyHzS1rTWQ==} + '@fluidframework/server-services@8.0.0-347501': + resolution: {integrity: sha512-LSvRlnbmhtcV4+iUzH4K+4VuRePqFkYxzF36l6wX2UK9cSJAuA8awDSj3H3OSH8mNa2Qq+67M90Oj1Xp7M+bgg==} - '@fluidframework/server-test-utils@6.0.0-340759': - resolution: {integrity: sha512-mPPleCzwqz1DZFvf8Zk3xroQZGfzm2fqRsajXFymxO7b/ea8IE1l3i16ZB0m9jrmctq/PFzAGSosarJB1XdgxQ==} + '@fluidframework/server-test-utils@8.0.0-347501': + resolution: {integrity: sha512-vtd19R6BhicC8dPCt6zvjM6dPfu1DIOyv/gJty2l7Rf/Kjya/iIn4ibB4jTm1IaJ5DjGxp58qC6DUnAUHMJmcw==} '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} @@ -4315,6 +4315,9 @@ packages: nan@2.19.0: resolution: {integrity: sha512-nO1xXxfh/RWNxfd/XPfbIfFk5vgLsAxUR9y5O0cHMJu/AW9U95JLXqthYHjEp+8gQ5p96K9jUp8nbVOxCdRbtw==} + nan@2.23.0: + resolution: {integrity: sha512-1UxuyYGdoQHcGg87Lkqm3FzefucTa0NAiOcuRsDmysep3c1LVCRK2krrUDafMWtjSG04htvAmvg96+SDknOmgQ==} + napi-build-utils@1.0.2: resolution: {integrity: sha512-ONmRUqK7zj7DWX0D9ADe03wbwOBZxNAfF20PlGfCWQcD3+/MakShIHrMqx9YwPTfxDdF1zLeL+RGZiR9kGMLdg==} @@ -4381,8 +4384,8 @@ packages: engines: {node: ^12.13 || ^14.13 || >=16} hasBin: true - node-rdkafka@3.0.1: - resolution: {integrity: sha512-USTFu7ylRj+fEiGz0hA92GWSqmX/hu/xSTqtgmInPPmh5zKhjauTciRjDEG3yK5m6yChwyHKQTIgmr56DfhiaQ==} + node-rdkafka@3.4.1: + resolution: {integrity: sha512-qWEjyeZmCNtc7MVUucl9xRsSSXdmMS3lQOiQU15fXIFaLK0K3OZr5T6VTjAAIgz+yoATvaCBDXmMc4RiK65eTg==} engines: {node: '>=16'} node-releases@2.0.14: @@ -6882,22 +6885,22 @@ snapshots: - supports-color - typescript - '@fluidframework/gitresources@6.0.0-340759': {} + '@fluidframework/gitresources@8.0.0-347501': {} - '@fluidframework/protocol-base@6.0.0-340759': + '@fluidframework/protocol-base@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 events_pkg: events@3.3.0 '@fluidframework/protocol-definitions@3.2.0': {} - '@fluidframework/server-services-client@6.0.0-340759': + '@fluidframework/server-services-client@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 - '@fluidframework/protocol-base': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 + '@fluidframework/protocol-base': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 axios: 1.8.4(debug@4.4.3) crc-32: 1.2.0 @@ -6910,13 +6913,13 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-services-core@6.0.0-340759': + '@fluidframework/server-services-core@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 '@types/nconf': 0.10.3 '@types/node': 18.19.3 debug: 4.4.3(supports-color@8.1.1) @@ -6925,12 +6928,12 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-services-ordering-kafkanode@6.0.0-340759': + '@fluidframework/server-services-ordering-kafkanode@8.0.0-347501': dependencies: - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-ordering-zookeeper': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-ordering-zookeeper': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 events_pkg: events@3.3.0 kafka-node: 5.0.0 nconf: 0.12.0 @@ -6938,37 +6941,37 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-services-ordering-rdkafka@6.0.0-340759': + '@fluidframework/server-services-ordering-rdkafka@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 - '@fluidframework/server-services-utils': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 + '@fluidframework/server-services-utils': 8.0.0-347501 events_pkg: events@3.3.0 nconf: 0.12.0 - node-rdkafka: 3.0.1 + node-rdkafka: 3.4.1 sillyname: 0.1.0 transitivePeerDependencies: - supports-color - '@fluidframework/server-services-ordering-zookeeper@6.0.0-340759': + '@fluidframework/server-services-ordering-zookeeper@8.0.0-347501': dependencies: - '@fluidframework/server-services-core': 6.0.0-340759 + '@fluidframework/server-services-core': 8.0.0-347501 zookeeper: 5.6.0 transitivePeerDependencies: - supports-color - '@fluidframework/server-services-shared@6.0.0-340759': + '@fluidframework/server-services-shared@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 - '@fluidframework/protocol-base': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 + '@fluidframework/protocol-base': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 - '@fluidframework/server-services-utils': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 + '@fluidframework/server-services-utils': 8.0.0-347501 '@socket.io/redis-adapter': 8.3.0(socket.io-adapter@2.5.5) '@socket.io/sticky': 1.0.4 body-parser: 1.20.3 @@ -6991,7 +6994,7 @@ snapshots: - supports-color - utf-8-validate - '@fluidframework/server-services-telemetry@6.0.0-340759': + '@fluidframework/server-services-telemetry@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 json-stringify-safe: 5.0.1 @@ -6999,12 +7002,12 @@ snapshots: serialize-error: 8.1.0 uuid: 11.1.0 - '@fluidframework/server-services-utils@6.0.0-340759': + '@fluidframework/server-services-utils@8.0.0-347501': dependencies: '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 debug: 4.4.3(supports-color@8.1.1) express: 4.21.2 ioredis: 5.6.1 @@ -7021,17 +7024,17 @@ snapshots: transitivePeerDependencies: - supports-color - '@fluidframework/server-services@6.0.0-340759': + '@fluidframework/server-services@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-ordering-kafkanode': 6.0.0-340759 - '@fluidframework/server-services-ordering-rdkafka': 6.0.0-340759 - '@fluidframework/server-services-shared': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 - '@fluidframework/server-services-utils': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-ordering-kafkanode': 8.0.0-347501 + '@fluidframework/server-services-ordering-rdkafka': 8.0.0-347501 + '@fluidframework/server-services-shared': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 + '@fluidframework/server-services-utils': 8.0.0-347501 '@socket.io/redis-emitter': 4.1.1 '@types/lodash': 4.17.7 amqplib: 0.10.3 @@ -7053,15 +7056,15 @@ snapshots: - supports-color - utf-8-validate - '@fluidframework/server-test-utils@6.0.0-340759': + '@fluidframework/server-test-utils@8.0.0-347501': dependencies: '@fluidframework/common-utils': 3.1.0 - '@fluidframework/gitresources': 6.0.0-340759 - '@fluidframework/protocol-base': 6.0.0-340759 + '@fluidframework/gitresources': 8.0.0-347501 + '@fluidframework/protocol-base': 8.0.0-347501 '@fluidframework/protocol-definitions': 3.2.0 - '@fluidframework/server-services-client': 6.0.0-340759 - '@fluidframework/server-services-core': 6.0.0-340759 - '@fluidframework/server-services-telemetry': 6.0.0-340759 + '@fluidframework/server-services-client': 8.0.0-347501 + '@fluidframework/server-services-core': 8.0.0-347501 + '@fluidframework/server-services-telemetry': 8.0.0-347501 '@types/ioredis-mock': 8.2.6(ioredis@5.6.1) assert: 2.0.0 debug: 4.4.3(supports-color@8.1.1) @@ -11677,6 +11680,8 @@ snapshots: nan@2.19.0: {} + nan@2.23.0: {} + napi-build-utils@1.0.2: optional: true @@ -11754,10 +11759,10 @@ snapshots: - bluebird - supports-color - node-rdkafka@3.0.1: + node-rdkafka@3.4.1: dependencies: bindings: 1.5.0 - nan: 2.19.0 + nan: 2.23.0 node-releases@2.0.14: {} diff --git a/server/routerlicious/packages/lambdas/src/index.ts b/server/routerlicious/packages/lambdas/src/index.ts index 4b6950a61916..30ef4d6d807b 100644 --- a/server/routerlicious/packages/lambdas/src/index.ts +++ b/server/routerlicious/packages/lambdas/src/index.ts @@ -29,10 +29,12 @@ export { type ISummaryReader, type ISummaryWriter, type ISummaryWriteResponse, + type ISummaryWriterFactory, ScribeLambda, ScribeLambdaFactory, SummaryReader, SummaryWriter, + SummaryWriterFactory, } from "./scribe"; export { ScriptoriumLambda, ScriptoriumLambdaFactory } from "./scriptorium"; export { diff --git a/server/routerlicious/packages/lambdas/src/nexus/connect.ts b/server/routerlicious/packages/lambdas/src/nexus/connect.ts index 8743ea10dd93..2a36189d3428 100644 --- a/server/routerlicious/packages/lambdas/src/nexus/connect.ts +++ b/server/routerlicious/packages/lambdas/src/nexus/connect.ts @@ -27,6 +27,7 @@ import { type ICollaborationSessionClient, clusterDrainingRetryTimeInMs, type IDenyList, + StageTrace, } from "@fluidframework/server-services-core"; import { CommonProperties, @@ -47,7 +48,7 @@ import type { } from "./interfaces"; import { ProtocolVersions, checkProtocolVersion } from "./protocol"; import { checkThrottleAndUsage, getSocketConnectThrottleId } from "./throttleAndUsage"; -import { StageTrace, sampleMessages } from "./trace"; +import { sampleMessages } from "./trace"; import { getMessageMetadata, handleServerErrorAndConvertToNetworkError, diff --git a/server/routerlicious/packages/lambdas/src/nexus/index.ts b/server/routerlicious/packages/lambdas/src/nexus/index.ts index 2943e67c9c50..c578a70352df 100644 --- a/server/routerlicious/packages/lambdas/src/nexus/index.ts +++ b/server/routerlicious/packages/lambdas/src/nexus/index.ts @@ -569,9 +569,9 @@ export function configureWebSocketServices( /** * @param contentBatches - typed as `unknown` array as it comes from wire and has not been validated. * v1 signals are expected to be an array of strings (Json.stringified `ISignalEnvelope`s from - * [Container.submitSignal](https://github.com/microsoft/FluidFramework/blob/ccb26baf65be1cbe3f708ec0fe6887759c25be6d/packages/loader/container-loader/src/container.ts#L2292-L2294) + * {@link https://github.com/microsoft/FluidFramework/blob/ccb26baf65be1cbe3f708ec0fe6887759c25be6d/packages/loader/container-loader/src/container.ts#L2292-L2294 | Container.submitSignal}4) * and sent via - * [DocumentDeltaConnection.emitMessages](https://github.com/microsoft/FluidFramework/blob/ccb26baf65be1cbe3f708ec0fe6887759c25be6d/packages/drivers/driver-base/src/documentDeltaConnection.ts#L313C1-L321C4)), + * {@link https://github.com/microsoft/FluidFramework/blob/ccb26baf65be1cbe3f708ec0fe6887759c25be6d/packages/drivers/driver-base/src/documentDeltaConnection.ts#L313C1-L321C4 | DocumentDeltaConnection.emitMessages}4)), * but actual content is passed-thru and not decoded. * * v2 signals are expected to be an array of `ISentSignalMessage` objects. diff --git a/server/routerlicious/packages/lambdas/src/nexus/trace.ts b/server/routerlicious/packages/lambdas/src/nexus/trace.ts index f75c5e55ff57..dc1eda38a620 100644 --- a/server/routerlicious/packages/lambdas/src/nexus/trace.ts +++ b/server/routerlicious/packages/lambdas/src/nexus/trace.ts @@ -3,7 +3,6 @@ * Licensed under the MIT License. */ -import { performance } from "@fluidframework/common-utils"; import type { IDocumentMessage } from "@fluidframework/protocol-definitions"; import { getRandomInt } from "@fluidframework/server-services-client"; import { DefaultServiceConfiguration } from "@fluidframework/server-services-core"; @@ -52,31 +51,3 @@ export function addNexusMessageTrace( return message; } - -interface IStageTrace { - /** - * Name of the Stage. - */ - stage: string; - /** - * Start time of the stage relative to the previous stage's start time. - */ - ts: number; -} -export class StageTrace { - private readonly traces: IStageTrace[] = []; - private lastStampedTraceTime: number = performance.now(); - constructor(initialStage?: T) { - if (initialStage) { - this.traces.push({ stage: initialStage.toString(), ts: 0 }); - } - } - public get trace(): IStageTrace[] { - return this.traces; - } - public stampStage(stage: T): void { - const now = performance.now(); - this.traces.push({ stage: stage.toString(), ts: now - this.lastStampedTraceTime }); - this.lastStampedTraceTime = now; - } -} diff --git a/server/routerlicious/packages/lambdas/src/scribe/index.ts b/server/routerlicious/packages/lambdas/src/scribe/index.ts index 9ba3f6e47c1c..5b6f874d9383 100644 --- a/server/routerlicious/packages/lambdas/src/scribe/index.ts +++ b/server/routerlicious/packages/lambdas/src/scribe/index.ts @@ -10,9 +10,10 @@ export type { IPendingMessageReader, ISummaryReader, ISummaryWriter, + ISummaryWriterFactory, ISummaryWriteResponse, } from "./interfaces"; export { ScribeLambda } from "./lambda"; export { ScribeLambdaFactory } from "./lambdaFactory"; export { SummaryReader } from "./summaryReader"; -export { SummaryWriter } from "./summaryWriter"; +export { SummaryWriter, SummaryWriterFactory } from "./summaryWriter"; diff --git a/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts b/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts index 991399ccaed1..f05c93c7ff94 100644 --- a/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts +++ b/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts @@ -9,7 +9,13 @@ import type { ISequencedDocumentAugmentedMessage, ISequencedDocumentMessage, } from "@fluidframework/protocol-definitions"; -import type { IScribe, ISequencedOperationMessage } from "@fluidframework/server-services-core"; +import type { IGitManager } from "@fluidframework/server-services-client"; +import type { + ICollection, + IDeltaService, + IScribe, + ISequencedOperationMessage, +} from "@fluidframework/server-services-core"; /** * @internal @@ -90,3 +96,17 @@ export interface ICheckpointManager { delete(sequenceNumber: number, lte: boolean): Promise; } + +export interface ISummaryWriterFactory { + create( + documentId: string, + tenantId: string, + gitManager: IGitManager, + deltaManager: IDeltaService, + messageCollection: ICollection, + enableWholeSummaryUpload: boolean, + lastSummaryMessages: ISequencedDocumentMessage[], + getDeltasViaAlfred: boolean, + maxLogtailLength: number, + ): ISummaryWriter; +} diff --git a/server/routerlicious/packages/lambdas/src/scribe/lambdaFactory.ts b/server/routerlicious/packages/lambdas/src/scribe/lambdaFactory.ts index bea47267d715..85ca104ca362 100644 --- a/server/routerlicious/packages/lambdas/src/scribe/lambdaFactory.ts +++ b/server/routerlicious/packages/lambdas/src/scribe/lambdaFactory.ts @@ -36,11 +36,10 @@ import { import { NoOpLambda, createSessionMetric, isDocumentValid, isDocumentSessionValid } from "../utils"; import { CheckpointManager } from "./checkpointManager"; -import type { ILatestSummaryState } from "./interfaces"; +import type { ILatestSummaryState, ISummaryWriterFactory } from "./interfaces"; import { ScribeLambda } from "./lambda"; import { PendingMessageReader } from "./pendingMessageReader"; import { SummaryReader } from "./summaryReader"; -import { SummaryWriter } from "./summaryWriter"; import { getClientIds, initializeProtocol, isScribeCheckpointQuorumScrubbed } from "./utils"; const DefaultScribe: IScribe = { @@ -87,6 +86,7 @@ export class ScribeLambdaFactory private readonly kafkaCheckpointOnReprocessingOp: boolean, private readonly maxLogtailLength: number, private readonly maxPendingCheckpointMessagesLength: number, + private readonly summaryWriterFactory: ISummaryWriterFactory, ) { super(); } @@ -314,7 +314,7 @@ export class ScribeLambdaFactory const protocolHandler = initializeProtocol(lastCheckpoint.protocolState); const lastSummaryMessages = latestSummary.messages; - const summaryWriter = new SummaryWriter( + const summaryWriter = this.summaryWriterFactory.create( tenantId, documentId, gitManager, diff --git a/server/routerlicious/packages/lambdas/src/scribe/summaryWriter.ts b/server/routerlicious/packages/lambdas/src/scribe/summaryWriter.ts index 0a17cf29d352..9e0dea53c0ef 100644 --- a/server/routerlicious/packages/lambdas/src/scribe/summaryWriter.ts +++ b/server/routerlicious/packages/lambdas/src/scribe/summaryWriter.ts @@ -46,7 +46,33 @@ import { } from "@fluidframework/server-services-telemetry"; import safeStringify from "json-stringify-safe"; -import type { ISummaryWriteResponse, ISummaryWriter } from "./interfaces"; +import type { ISummaryWriteResponse, ISummaryWriter, ISummaryWriterFactory } from "./interfaces"; + +export class SummaryWriterFactory implements ISummaryWriterFactory { + public create( + tenantId: string, + documentId: string, + gitManager: IGitManager, + deltaManager: IDeltaService, + messageCollection: ICollection, + enableWholeSummaryUpload: boolean, + lastSummaryMessages: ISequencedDocumentMessage[], + getDeltasViaAlfred: boolean, + maxLogtailLength: number, + ): ISummaryWriter { + return new SummaryWriter( + tenantId, + documentId, + gitManager, + deltaManager, + messageCollection, + enableWholeSummaryUpload, + lastSummaryMessages, + getDeltasViaAlfred, + maxLogtailLength, + ); + } +} /** * Git specific implementation of ISummaryWriter @@ -55,8 +81,8 @@ import type { ISummaryWriteResponse, ISummaryWriter } from "./interfaces"; export class SummaryWriter implements ISummaryWriter { private readonly lumberProperties: Record; constructor( - private readonly tenantId: string, - private readonly documentId: string, + protected readonly tenantId: string, + protected readonly documentId: string, private readonly summaryStorage: IGitManager, private readonly deltaService: IDeltaService | undefined, private readonly opStorage: ICollection | undefined, @@ -100,6 +126,18 @@ export class SummaryWriter implements ISummaryWriter { ): Promise { const clientSummaryMetric = Lumberjack.newLumberMetric(LumberEventName.ClientSummary); this.setSummaryProperties(clientSummaryMetric, op, isEphemeralContainer); + if (!(await this.isDocumentValid(isEphemeralContainer))) { + clientSummaryMetric.error(`Document is not valid to accept summaries`); + return { + message: { + message: `Document is not valid to accept summaries`, + summaryProposal: { + summarySequenceNumber: op.sequenceNumber, + }, + }, + status: false, + }; + } const content = JSON.parse(op.contents as string) as ISummaryContent; try { // The summary must reference the existing summary to be valid. This guards against accidental sends of @@ -408,6 +446,10 @@ export class SummaryWriter implements ISummaryWriter { ): Promise { const serviceSummaryMetric = Lumberjack.newLumberMetric(LumberEventName.ServiceSummary); this.setSummaryProperties(serviceSummaryMetric, op, isEphemeralContainer); + if (!(await this.isDocumentValid(isEphemeralContainer))) { + serviceSummaryMetric.error(`Document is not valid to accept summaries`); + return false; + } try { const existingRef = await requestWithRetry( async () => this.summaryStorage.getRef(encodeURIComponent(this.documentId)), @@ -579,6 +621,14 @@ export class SummaryWriter implements ISummaryWriter { } } + /** + * Validates whether the document is in a valid state to accept summaries. + * @returns A boolean indicating whether the document is valid. + */ + protected async isDocumentValid(isEphemeralContainer: boolean | undefined): Promise { + return true; + } + private setSummaryProperties( summaryMetric: Lumber, op: ISequencedDocumentAugmentedMessage, diff --git a/server/routerlicious/packages/lambdas/src/test/scribe/lambda.spec.ts b/server/routerlicious/packages/lambdas/src/test/scribe/lambda.spec.ts index 5f3e8344d384..162a866460c1 100644 --- a/server/routerlicious/packages/lambdas/src/test/scribe/lambda.spec.ts +++ b/server/routerlicious/packages/lambdas/src/test/scribe/lambda.spec.ts @@ -31,6 +31,7 @@ import _ from "lodash"; import Sinon from "sinon"; import { ScribeLambda } from "../../scribe/lambda"; import { ScribeLambdaFactory } from "../../scribe/lambdaFactory"; +import { SummaryWriterFactory } from "../../scribe"; describe("Routerlicious", () => { describe("Scribe", () => { @@ -161,6 +162,7 @@ describe("Routerlicious", () => { true, 2000, 2000, + new SummaryWriterFactory(), ); testContext = new TestContext(); diff --git a/server/routerlicious/packages/routerlicious/src/scribe/index.ts b/server/routerlicious/packages/routerlicious/src/scribe/index.ts index 03687a737fe7..bd73285c531d 100644 --- a/server/routerlicious/packages/routerlicious/src/scribe/index.ts +++ b/server/routerlicious/packages/routerlicious/src/scribe/index.ts @@ -3,7 +3,11 @@ * Licensed under the MIT License. */ -import { ScribeLambdaFactory } from "@fluidframework/server-lambdas"; +import { + ISummaryWriterFactory, + ScribeLambdaFactory, + SummaryWriterFactory, +} from "@fluidframework/server-lambdas"; import { createDocumentRouter } from "@fluidframework/server-routerlicious-base"; import { createProducer, @@ -180,6 +184,9 @@ export async function scribeCreate( localCheckpointEnabled, ); + const summaryWriterFactory: ISummaryWriterFactory = + customizations?.summaryWriterFactory ?? new SummaryWriterFactory(); + return new ScribeLambdaFactory( operationsDbManager, documentRepository, @@ -198,6 +205,7 @@ export async function scribeCreate( kafkaCheckpointOnReprocessingOp, maxLogtailLength, maxPendingCheckpointMessagesLength, + summaryWriterFactory, ); } diff --git a/server/routerlicious/packages/services-client/api-report/server-services-client.api.md b/server/routerlicious/packages/services-client/api-report/server-services-client.api.md index 6cfd0d7ec790..be9762b84fb6 100644 --- a/server/routerlicious/packages/services-client/api-report/server-services-client.api.md +++ b/server/routerlicious/packages/services-client/api-report/server-services-client.api.md @@ -174,6 +174,29 @@ export class GitManager implements IGitManager { write(branch: string, inputTree: api.ITree, parents: string[], message: string): Promise; } +// @internal +export type GitManagerConfigDecorator = (config: IGitManagerConfig, context: { + tenantId: string; + documentId: string; + storageName?: string; + isEphemeralContainer: boolean; + accessToken: string; + baseUrl: string; +}) => IGitManagerConfig; + +// @internal +export const GitManagerConfigDecorators: { + withCustom: (customConfig: Partial | ((config: IGitManagerConfig, context: { + tenantId: string; + documentId: string; + storageName?: string; + isEphemeralContainer: boolean; + accessToken: string; + baseUrl: string; + }) => Partial)) => GitManagerConfigDecorator; + compose: (...decorators: GitManagerConfigDecorator[]) => GitManagerConfigDecorator; +}; + // @internal export class Heap { constructor(comparator: IHeapComparator); @@ -357,6 +380,22 @@ export interface IGitManager { write(branch: string, inputTree: api.ITree, parents: string[], message: string): Promise; } +// @internal +export interface IGitManagerConfig { + defaultHeaders?: RawAxiosRequestHeaders; + defaultQueryString?: Record; + getCorrelationId?: () => string | undefined; + getDefaultHeaders?: () => RawAxiosRequestHeaders; + getServiceName?: () => string; + getTelemetryProperties?: () => Record; + logHttpMetrics?: (requestProps: any) => void; + maxBodyLength?: number; + maxContentLength?: number; + refreshDefaultHeaders?: () => RawAxiosRequestHeaders; + refreshDefaultQueryString?: () => Record; + refreshTokenIfNeeded?: (authorizationHeader: RawAxiosRequestHeaders) => Promise; +} + // @internal export interface IGitService { // (undocumented) @@ -469,7 +508,7 @@ export interface ISummaryTree extends ISummaryTree_2 { // @internal export interface ISummaryUploadManager { - writeSummaryTree(summaryTree: api.ISummaryTree, parentHandle: string, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number): Promise; + writeSummaryTree(summaryTree: api.ISummaryTree, parentHandle: string, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, initial?: boolean, summaryTimeStr?: string): Promise; } // @internal @@ -707,7 +746,7 @@ export function setupAxiosInterceptorsForAbortSignals(getAbortController: () => export class SummaryTreeUploadManager implements ISummaryUploadManager { constructor(manager: IGitManager, blobsShaCache: Map, getPreviousFullSnapshot: (parentHandle: string) => Promise); // (undocumented) - writeSummaryTree(summaryTree: ISummaryTree_2, parentHandle: string, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, initial?: boolean): Promise; + writeSummaryTree(summaryTree: ISummaryTree_2, parentHandle: string, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, initial?: boolean, summaryTimeStr?: string): Promise; } // @internal @@ -735,7 +774,7 @@ export type WholeSummaryTreeValue = IWholeSummaryTree | IWholeSummaryBlob; export class WholeSummaryUploadManager implements ISummaryUploadManager { constructor(manager: IGitManager); // (undocumented) - writeSummaryTree(summaryTree: ISummaryTree, parentHandle: string | undefined, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, initial?: boolean): Promise; + writeSummaryTree(summaryTree: ISummaryTree, parentHandle: string | undefined, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, initial?: boolean, summaryTimeStr?: string): Promise; } // (No @packageDocumentation comment for this package) diff --git a/server/routerlicious/packages/services-client/src/index.ts b/server/routerlicious/packages/services-client/src/index.ts index 4b3882e667da..c6ea94601cf1 100644 --- a/server/routerlicious/packages/services-client/src/index.ts +++ b/server/routerlicious/packages/services-client/src/index.ts @@ -65,16 +65,19 @@ export { generateServiceProtocolEntries, } from "./scribeHelper"; export type { + GitManagerConfigDecorator, ICreateRefParamsExternal, IExternalWriterConfig, IGetRefParamsExternal, IGitCache, IGitManager, + IGitManagerConfig, IGitService, IHistorian, IPatchRefParamsExternal, ISummaryUploadManager, } from "./storage"; +export { GitManagerConfigDecorators } from "./storage"; export type { ExtendedSummaryObject, IEmbeddedSummaryHandle, diff --git a/server/routerlicious/packages/services-client/src/storage.ts b/server/routerlicious/packages/services-client/src/storage.ts index 4e0e5a581451..468badfa7223 100644 --- a/server/routerlicious/packages/services-client/src/storage.ts +++ b/server/routerlicious/packages/services-client/src/storage.ts @@ -5,6 +5,7 @@ import type * as git from "@fluidframework/gitresources"; import type * as api from "@fluidframework/protocol-definitions"; +import type { RawAxiosRequestHeaders } from "axios"; import type { IWholeSummaryPayload, @@ -102,6 +103,202 @@ export interface IHistorian extends IGitService { getFullTree(sha: string): Promise; } +/** + * Filters out potentially dangerous headers that could compromise security + * @internal + */ +function filterSafeHeaders(headers: RawAxiosRequestHeaders): RawAxiosRequestHeaders { + const dangerousHeaders = [ + "authorization", + "cookie", + "x-auth-token", + "bearer", + "x-auth", + "auth", + "token", + "storagename", + ]; + + const filtered: RawAxiosRequestHeaders = {}; + for (const [key, value] of Object.entries(headers)) { + const isSecurityHeader = dangerousHeaders.some((dangerous) => + key.toLowerCase().includes(dangerous.toLowerCase()), + ); + + if (!isSecurityHeader) { + filtered[key] = value; + } + } + + return filtered; +} + +/** + * Configuration options for customizing the GitManager creation + * @internal + */ +export interface IGitManagerConfig { + /** Default query string parameters */ + defaultQueryString?: Record; + /** Maximum body length for requests */ + maxBodyLength?: number; + /** Maximum content length for requests */ + maxContentLength?: number; + /** Default headers to include */ + defaultHeaders?: RawAxiosRequestHeaders; + /** Custom function to get default headers */ + getDefaultHeaders?: () => RawAxiosRequestHeaders; + /** Custom function to refresh default query string */ + refreshDefaultQueryString?: () => Record; + /** Custom function to refresh default headers */ + refreshDefaultHeaders?: () => RawAxiosRequestHeaders; + /** Custom function to refresh tokens when needed */ + refreshTokenIfNeeded?: ( + authorizationHeader: RawAxiosRequestHeaders, + ) => Promise; + /** Custom function to get correlation ID */ + getCorrelationId?: () => string | undefined; + /** Custom function to get telemetry properties */ + getTelemetryProperties?: () => Record; + /** Custom function to log HTTP metrics */ + logHttpMetrics?: (requestProps: any) => void; + /** Custom function to get service name */ + getServiceName?: () => string; +} + +/** + * Decorator function type for customizing GitManager configuration + * @internal + */ +export type GitManagerConfigDecorator = ( + config: IGitManagerConfig, + context: { + tenantId: string; + documentId: string; + storageName?: string; + isEphemeralContainer: boolean; + accessToken: string; + baseUrl: string; + }, +) => IGitManagerConfig; + +/** + * Utility decorator implementations for common use cases + * + * ⚠️ SECURITY NOTE: This decorator implements security protections to prevent + * malicious code injection. Critical security functions (token refresh, + * correlation ID, telemetry) are immutable and cannot be overridden. + * Authorization headers and security-related headers are filtered and protected. + * + * @internal + */ +export const GitManagerConfigDecorators = { + /** + * Applies custom configuration overrides with built-in security protections. + * + * 🔒 Security Features: + * - Filters out dangerous headers (authorization, auth tokens, cookies, etc.) + * - Protects critical security functions from being overridden + * - Preserves system-generated authorization and routing headers + * - Prevents token refresh logic manipulation + * + * @param customConfig - Partial configuration object or a function that receives + * the current config and context to return custom overrides + * + * @example Simple overrides (security-filtered): + * ```typescript + * GitManagerConfigDecorators.withCustom({ + * defaultHeaders: { + * 'X-Custom-Header': 'value', // ✅ Safe custom header + * 'Authorization': 'Bearer evil', // ❌ Filtered out for security + * }, + * maxBodyLength: 2000 * 1024, // ✅ Safe limit override + * logHttpMetrics: (props) => console.log('Custom metrics:', props), // ✅ Allowed + * }) + * ``` + * + * @example Context-aware overrides: + * ```typescript + * GitManagerConfigDecorators.withCustom((config, context) => ({ + * defaultHeaders: { + * 'X-Tenant-ID': context.tenantId, // ✅ Safe tenant info + * 'X-Document-Type': context.isEphemeralContainer ? 'ephemeral' : 'persistent', + * }, + * maxBodyLength: context.isEphemeralContainer ? 100 * 1024 : 1000 * 1024, + * })) + * ``` + * + * @example Protected functions (these are immutable for security): + * ```typescript + * // ❌ These overrides will be ignored for security: + * GitManagerConfigDecorators.withCustom({ + * refreshTokenIfNeeded: evilTokenStealer, // Ignored - security protected + * getCorrelationId: () => 'hacked', // Ignored - security protected + * getTelemetryProperties: evilLogger, // Ignored - security protected + * }) + * ``` + */ + withCustom: + ( + customConfig: + | Partial + | (( + config: IGitManagerConfig, + context: { + tenantId: string; + documentId: string; + storageName?: string; + isEphemeralContainer: boolean; + accessToken: string; + baseUrl: string; + }, + ) => Partial), + ): GitManagerConfigDecorator => + (config, context) => { + const overrides = + typeof customConfig === "function" ? customConfig(config, context) : customConfig; + + // Apply customizations with security filtering + const secureConfig = { + ...config, + ...overrides, + // Handle nested objects properly by merging them with security filtering + defaultHeaders: { + ...config.defaultHeaders, + ...filterSafeHeaders(overrides.defaultHeaders || {}), + }, + defaultQueryString: { + ...config.defaultQueryString, + ...overrides.defaultQueryString, + }, + }; + + // 🔒 IMMUTABLE SECURITY LAYER - These critical functions cannot be overridden + secureConfig.refreshTokenIfNeeded = config.refreshTokenIfNeeded; + secureConfig.getCorrelationId = config.getCorrelationId; + secureConfig.getTelemetryProperties = config.getTelemetryProperties; + secureConfig.getServiceName = config.getServiceName; + + // Ensure authorization and other security-critical headers are preserved + if (config.defaultHeaders?.Authorization) { + secureConfig.defaultHeaders.Authorization = config.defaultHeaders.Authorization; + } + if (config.defaultHeaders?.StorageName) { + secureConfig.defaultHeaders.StorageName = config.defaultHeaders.StorageName; + } + + return secureConfig; + }, + + /** + * Composes multiple decorators into one + */ + compose: + (...decorators: GitManagerConfigDecorator[]): GitManagerConfigDecorator => + (config, context) => + decorators.reduce((acc, decorator) => decorator(acc, context), config), +}; + /** * @internal */ @@ -151,5 +348,7 @@ export interface ISummaryUploadManager { parentHandle: string, summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, + initial?: boolean, + summaryTimeStr?: string, ): Promise; } diff --git a/server/routerlicious/packages/services-client/src/summaryTreeUploadManager.ts b/server/routerlicious/packages/services-client/src/summaryTreeUploadManager.ts index 5ed386648aff..0a2b482aa5bf 100644 --- a/server/routerlicious/packages/services-client/src/summaryTreeUploadManager.ts +++ b/server/routerlicious/packages/services-client/src/summaryTreeUploadManager.ts @@ -42,6 +42,7 @@ export class SummaryTreeUploadManager implements ISummaryUploadManager { summaryType: IWholeSummaryPayloadType, sequenceNumber?: number, initial?: boolean, + summaryTimeStr?: string, ): Promise { const previousFullSnapshot = await this.getPreviousFullSnapshot(parentHandle); return this.writeSummaryTreeCore(summaryTree, previousFullSnapshot ?? undefined); diff --git a/server/routerlicious/packages/services-client/src/wholeSummaryUploadManager.ts b/server/routerlicious/packages/services-client/src/wholeSummaryUploadManager.ts index 8b4aba99e43d..96f17375108f 100644 --- a/server/routerlicious/packages/services-client/src/wholeSummaryUploadManager.ts +++ b/server/routerlicious/packages/services-client/src/wholeSummaryUploadManager.ts @@ -24,6 +24,7 @@ export class WholeSummaryUploadManager implements ISummaryUploadManager { summaryType: IWholeSummaryPayloadType, sequenceNumber: number = 0, initial: boolean = false, + summaryTimeStr?: string, ): Promise { const id = await this.writeSummaryTreeCore( parentHandle, @@ -31,6 +32,7 @@ export class WholeSummaryUploadManager implements ISummaryUploadManager { summaryType, sequenceNumber, initial, + summaryTimeStr, ); if (!id) { throw new Error(`Failed to write summary tree`); @@ -44,6 +46,7 @@ export class WholeSummaryUploadManager implements ISummaryUploadManager { type: IWholeSummaryPayloadType, sequenceNumber: number, initial: boolean, + summaryTimeStr?: string, ): Promise { const snapshotTree = convertSummaryTreeToWholeSummaryTree( parentHandle, @@ -56,6 +59,7 @@ export class WholeSummaryUploadManager implements ISummaryUploadManager { message: `${type} summary upload`, sequenceNumber, type, + summaryTime: summaryTimeStr, }; return this.manager.createSummary(snapshotPayload, initial).then((response) => response.id); diff --git a/server/routerlicious/packages/services-core/src/document.ts b/server/routerlicious/packages/services-core/src/document.ts index 50f8001f33ab..b2d7dc49f8ce 100644 --- a/server/routerlicious/packages/services-core/src/document.ts +++ b/server/routerlicious/packages/services-core/src/document.ts @@ -34,6 +34,18 @@ export interface IDocumentStaticProperties { isEphemeralContainer?: boolean; } +/** + * @internal + */ +export interface IAdditionalQueryParams { + [key: string]: + | undefined + | string + | string[] + | IAdditionalQueryParams + | IAdditionalQueryParams[]; +} + /** * @internal */ @@ -64,6 +76,7 @@ export interface IDocumentStorage { enableDiscovery: boolean, isEphemeralContainer: boolean, messageBrokerId?: string, + additionalQueryParams?: IAdditionalQueryParams, ): Promise; } diff --git a/server/routerlicious/packages/services-core/src/index.ts b/server/routerlicious/packages/services-core/src/index.ts index 477354b5b2dd..001d6cf5fd91 100644 --- a/server/routerlicious/packages/services-core/src/index.ts +++ b/server/routerlicious/packages/services-core/src/index.ts @@ -44,6 +44,7 @@ export type { IDocumentStaticProperties, IDocumentStorage, IScribe, + IAdditionalQueryParams, } from "./document"; export type { IDocumentManager } from "./documentManager"; export { EmptyTaskMessageSender } from "./emptyTaskMessageSender"; @@ -191,3 +192,4 @@ export type { IWebSocketTracker } from "./webSocketTracker"; export type { IReadinessCheck, IReadinessStatus, ICheck } from "./readinessCheck"; export type { IFluidAccessToken, IFluidAccessTokenGenerator } from "./fluidAccessTokenGenerator"; export type { IDenyList } from "./denyList"; +export { type IStageTrace, StageTrace } from "./trace"; diff --git a/server/routerlicious/packages/services-core/src/tenant.ts b/server/routerlicious/packages/services-core/src/tenant.ts index 5c038ddbcf21..d0989dd3282e 100644 --- a/server/routerlicious/packages/services-core/src/tenant.ts +++ b/server/routerlicious/packages/services-core/src/tenant.ts @@ -4,7 +4,10 @@ */ import type { IUser, ScopeType } from "@fluidframework/protocol-definitions"; -import type { IGitManager } from "@fluidframework/server-services-client"; +import type { + IGitManager, + GitManagerConfigDecorator, +} from "@fluidframework/server-services-client"; /** * @internal @@ -174,6 +177,7 @@ export interface ITenantManager { storageName?: string, includeDisabledTenant?: boolean, isEphemeralContainer?: boolean, + configDecorator?: GitManagerConfigDecorator, ): Promise; /** diff --git a/server/routerlicious/packages/services-core/src/trace.ts b/server/routerlicious/packages/services-core/src/trace.ts new file mode 100644 index 000000000000..487f39628503 --- /dev/null +++ b/server/routerlicious/packages/services-core/src/trace.ts @@ -0,0 +1,34 @@ +/*! + * Copyright (c) Microsoft Corporation and contributors. All rights reserved. + * Licensed under the MIT License. + */ + +import { performance } from "@fluidframework/common-utils"; + +export interface IStageTrace { + /** + * Name of the Stage. + */ + stage: string; + /** + * Start time of the stage relative to the previous stage's start time. + */ + ts: number; +} +export class StageTrace { + private readonly traces: IStageTrace[] = []; + private lastStampedTraceTime: number = performance.now(); + constructor(initialStage?: T) { + if (initialStage) { + this.traces.push({ stage: initialStage.toString(), ts: 0 }); + } + } + public get trace(): IStageTrace[] { + return this.traces; + } + public stampStage(stage: T): void { + const now = performance.now(); + this.traces.push({ stage: stage.toString(), ts: now - this.lastStampedTraceTime }); + this.lastStampedTraceTime = now; + } +} diff --git a/server/routerlicious/packages/services-shared/src/restLessServer.ts b/server/routerlicious/packages/services-shared/src/restLessServer.ts index 721de9e1af5c..dd945c8b44b0 100644 --- a/server/routerlicious/packages/services-shared/src/restLessServer.ts +++ b/server/routerlicious/packages/services-shared/src/restLessServer.ts @@ -22,7 +22,7 @@ type RequestField = undefined | string | string[]; interface IRestLessServerOptions { /** * Request body size limit in number of bytes or as a string to - * be passed to the [bytes](https://www.npmjs.com/package/bytes) library. + * be passed to th{@link https://www.npmjs.com/package/bytes | bytes}s) library. * Only verified when parsing request body as a stream. * Default: 1gb */ diff --git a/server/routerlicious/packages/services-shared/src/socketIoServer.ts b/server/routerlicious/packages/services-shared/src/socketIoServer.ts index 7702869740c3..41b8a02fa6cb 100644 --- a/server/routerlicious/packages/services-shared/src/socketIoServer.ts +++ b/server/routerlicious/packages/services-shared/src/socketIoServer.ts @@ -136,7 +136,7 @@ export interface ISocketIoServerConfig { */ pingPongLatencyTrackingAggregationThreshold: number; /** - * Whether to enable Socket.io [perMessageDeflate](https://socket.io/docs/v4/server-options/#permessagedeflate) option. + * Whether to enable Socket.i{@link https://socket.io/docs/v4/server-options/#permessagedeflate | perMessageDeflate}e) option. * Default is `true`. */ perMessageDeflate: boolean; @@ -156,7 +156,7 @@ class SocketIoServer implements core.IWebSocketServer { /** * Fluid Socket.io connection URL looks like: * "/socket.io/?documentId=[documentId]&tenantId=[tenantId]&EIO=[3/4]&transport=[websocket/polling]" - * [socket.handshake.query](https://socket.io/docs/v4/server-socket-instance/#sockethandshake) contains parsed query params. + * {@link https://socket.io/docs/v4/server-socket-instance/#sockethandshake | socket.handshake.query}e) contains parsed query params. * The following properties are used for **telemetry purposes only.** * These should **not** be used to identify the tenant and document associated with the socket connection * for real logic and access purposes without validating against the JWT access token. diff --git a/server/routerlicious/packages/services-shared/src/storage.ts b/server/routerlicious/packages/services-shared/src/storage.ts index dcc3b8798a30..fbba6c98db78 100644 --- a/server/routerlicious/packages/services-shared/src/storage.ts +++ b/server/routerlicious/packages/services-shared/src/storage.ts @@ -18,8 +18,11 @@ import { WholeSummaryUploadManager, type ISession, getGlobalTimeoutContext, + type IGitManager, + NetworkError, } from "@fluidframework/server-services-client"; import { + type IAdditionalQueryParams, type ICollection, type IDeliState, type IDocument, @@ -31,6 +34,7 @@ import { type IStorageNameAllocator, type ITenantManager, SequencedOperationType, + StageTrace, } from "@fluidframework/server-services-core"; import { BaseTelemetryProperties, @@ -38,20 +42,40 @@ import { getLumberBaseProperties, LumberEventName, Lumberjack, + type Lumber, } from "@fluidframework/server-services-telemetry"; +import { AsyncLocalStorageContextProvider } from "@fluidframework/server-services-utils"; import * as winston from "winston"; /** * @internal */ + +export enum DocCreationStage { + Started = "Started", + StorageNameAllocated = "StorageNameAllocated", + GitManagerCreated = "GitManagerCreated", + DocCreationCompleted = "DocCreationCompleted", + InitialSummaryUploaded = "InitialSummaryUploaded", + DocCreated = "DocCreated", +} + +// Basic protection against obvious attacks +const MAX_DEPTH = 10; // Prevent deep nesting attacks +const MAX_KEY_LENGTH = 200; // Prevent extremely long keys +const MAX_VALUE_LENGTH = 10000; // Prevent extremely long values +const MAX_OBJECT_SIZE = 100; // Prevent too many properties + export class DocumentStorage implements IDocumentStorage { + protected readonly createDocContext: AsyncLocalStorageContextProvider = + new AsyncLocalStorageContextProvider(); constructor( - private readonly documentRepository: IDocumentRepository, - private readonly tenantManager: ITenantManager, - private readonly enableWholeSummaryUpload: boolean, - private readonly opsCollection: ICollection, - private readonly storageNameAssigner: IStorageNameAllocator | undefined, - private readonly ephemeralDocumentTTLSec: number = 60 * 60 * 24, // 24 hours in seconds + protected readonly documentRepository: IDocumentRepository, + protected readonly tenantManager: ITenantManager, + protected readonly enableWholeSummaryUpload: boolean, + protected readonly opsCollection: ICollection, + protected readonly storageNameAssigner: IStorageNameAllocator | undefined, + protected readonly ephemeralDocumentTTLSec: number = 60 * 60 * 24, // 24 hours in seconds ) {} /** @@ -123,6 +147,21 @@ export class DocumentStorage implements IDocumentStorage { }; } + protected async getGitManager( + tenantId: string, + documentId: string, + storageName: string | undefined, + isEphemeralContainer: boolean, + ): Promise { + return this.tenantManager.getTenantGitManager( + tenantId, + documentId, + storageName, + false /* includeDisabledTenant */, + isEphemeralContainer, + ); + } + public async createDocument( tenantId: string, documentId: string, @@ -136,86 +175,139 @@ export class DocumentStorage implements IDocumentStorage { enableDiscovery: boolean = false, isEphemeralContainer: boolean = false, messageBrokerId?: string, + additionalQueryParams?: IAdditionalQueryParams, ): Promise { - const storageName = await this.storageNameAssigner?.assign(tenantId, documentId); - const gitManager = await this.tenantManager.getTenantGitManager( - tenantId, - documentId, - storageName, - false /* includeDisabledTenant */, - isEphemeralContainer, - ); - - const storageNameAssignerEnabled = !!this.storageNameAssigner; - const lumberjackProperties = { - ...getLumberBaseProperties(documentId, tenantId), - storageName, - enableWholeSummaryUpload: this.enableWholeSummaryUpload, - storageNameAssignerExists: storageNameAssignerEnabled, - [CommonProperties.isEphemeralContainer]: isEphemeralContainer, - }; - if (storageNameAssignerEnabled && !storageName) { - // Using a warning instead of an error just in case there are some outliers that we don't know about. - Lumberjack.warning( - "Failed to get storage name for new document.", + // Basic security validation for extensibility point + this.validateAdditionalQueryParams(additionalQueryParams); + return this.createDocContext.bindContext(additionalQueryParams ?? {}, async () => { + const createDocTrace = new StageTrace(DocCreationStage.Started); + const lumberjackProperties: Record = { + ...getLumberBaseProperties(documentId, tenantId), + enableWholeSummaryUpload: this.enableWholeSummaryUpload, + [CommonProperties.isEphemeralContainer]: isEphemeralContainer, + }; + let runTimeError; + const createDocMetric = Lumberjack.newLumberMetric( + LumberEventName.CreateDocument, lumberjackProperties, ); - } - - const protocolTree = this.createInitialProtocolTree(sequenceNumber, values); - const fullTree = this.createFullTree(appTree, protocolTree); + try { + const storageName = await this.storageNameAssigner?.assign(tenantId, documentId); + const storageNameAssignerEnabled = !!this.storageNameAssigner; + lumberjackProperties.storageName = storageName; + lumberjackProperties.storageNameAssignerEnabled = storageNameAssignerEnabled; + if (storageNameAssignerEnabled && !storageName) { + // Using a warning instead of an error just in case there are some outliers that we don't know about. + Lumberjack.warning( + "Failed to get storage name for new document.", + lumberjackProperties, + ); + } + createDocTrace.stampStage(DocCreationStage.StorageNameAllocated); + getGlobalTimeoutContext().checkTimeout(); - const blobsShaCache = new Map(); - const uploadManager = this.enableWholeSummaryUpload - ? new WholeSummaryUploadManager(gitManager) - : new SummaryTreeUploadManager(gitManager, blobsShaCache, async () => undefined); + const gitManager = await this.getGitManager( + tenantId, + documentId, + storageName, + isEphemeralContainer, + ); + + const protocolTree = this.createInitialProtocolTree(sequenceNumber, values); + const fullTree = this.createFullTree(appTree, protocolTree); + const blobsShaCache = new Map(); + const uploadManager = this.enableWholeSummaryUpload + ? new WholeSummaryUploadManager(gitManager) + : new SummaryTreeUploadManager( + gitManager, + blobsShaCache, + async () => undefined, + ); + + const summaryTimeStr = new Date().toISOString(); + + const dbResult: IDocumentDetails = await this.createDocumentCore( + createDocMetric, + lumberjackProperties, + createDocTrace, + uploadManager, + fullTree, + gitManager, + documentId, + summaryTimeStr, + sequenceNumber, + initialHash, + values, + ordererUrl, + historianUrl, + deltaStreamUrl, + messageBrokerId, + enableDiscovery, + tenantId, + storageName, + isEphemeralContainer, + additionalQueryParams, + ); + createDocTrace.stampStage(DocCreationStage.DocCreationCompleted); + return dbResult; + } catch (err: unknown) { + runTimeError = err; + throw err; + } finally { + createDocMetric.setProperty("createDocTrace", createDocTrace.trace); + if (runTimeError) { + createDocMetric.error("Failed created doc", runTimeError); + } else { + createDocMetric.success("Successfully created doc"); + } + } + }); + } + protected async createDocumentCore( + createDocMetric: Lumber, // Used by override + lumberjackProperties: Record, + createDocTrace: StageTrace, + uploadManager: WholeSummaryUploadManager | SummaryTreeUploadManager, + fullTree: ISummaryTree, + gitManager: IGitManager, + documentId: string, + summaryTimeStr: string, + sequenceNumber: number, + initialHash: string, + values: [string, ICommittedProposal][], + ordererUrl: string, + historianUrl: string, + deltaStreamUrl: string, + messageBrokerId: string | undefined, + enableDiscovery: boolean, + tenantId: string, + storageName: string | undefined, + isEphemeralContainer: boolean, + ): Promise { const initialSummaryUploadMetric = Lumberjack.newLumberMetric( LumberEventName.CreateDocInitialSummaryWrite, lumberjackProperties, ); + let initialSummaryVersionId: string; try { - const handle = await uploadManager.writeSummaryTree( - fullTree /* summaryTree */, - "" /* parentHandle */, - "container" /* summaryType */, - 0 /* sequenceNumber */, - true /* initial */, + const { summaryVersionId, summaryUploadMessage } = await this.uploadSummary( + uploadManager, + fullTree, + gitManager, + documentId, + summaryTimeStr, ); - - let initialSummaryUploadSuccessMessage = `Tree reference: ${JSON.stringify(handle)}`; - - if (!this.enableWholeSummaryUpload) { - const commitParams: ICreateCommitParams = { - author: { - date: new Date().toISOString(), - email: "dummy@microsoft.com", - name: "Routerlicious Service", - }, - message: "New document", - parents: [], - tree: handle, - }; - - const commit = await gitManager.createCommit(commitParams); - await gitManager.createRef(documentId, commit.sha); - initialSummaryUploadSuccessMessage += ` - Commit sha: ${JSON.stringify( - commit.sha, - )}`; - // In the case of ShreddedSummary Upload, summary version is always the commit sha. - initialSummaryVersionId = commit.sha; - } else { - // In the case of WholeSummary Upload, summary tree handle is actually commit sha or version id. - initialSummaryVersionId = handle; - } - initialSummaryUploadMetric.success(initialSummaryUploadSuccessMessage); + initialSummaryVersionId = summaryVersionId; + initialSummaryUploadMetric.success(summaryUploadMessage); } catch (error: any) { initialSummaryUploadMetric.error("Error during initial summary upload", error); throw error; } // Storage is known to take too long sometimes. Check timeout before continuing. + createDocTrace.stampStage(DocCreationStage.InitialSummaryUploaded); getGlobalTimeoutContext().checkTimeout(); const deli: IDeliState = { @@ -288,31 +380,160 @@ export class DocumentStorage implements IDocumentStorage { storageName, isEphemeralContainer, }; - const documentDbValue: IDocument & { ttl?: number } = { - ...document, - }; - if (isEphemeralContainer) { - documentDbValue.ttl = this.ephemeralDocumentTTLSec; - } + const documentDbValue: IDocument & { ttl?: number } = this.createDocumentDbValue( + document, + isEphemeralContainer, + ); + let dbResult: IDocumentDetails; try { - const result = await this.documentRepository.findOneOrCreate( + dbResult = await this.documentRepository.findOneOrCreate( { documentId, tenantId, }, documentDbValue, ); - createDocumentCollectionMetric.setProperty( - CommonProperties.isEphemeralContainer, - isEphemeralContainer, - ); + createDocTrace.stampStage(DocCreationStage.DocCreated); createDocumentCollectionMetric.success("Successfully created document"); - return result; } catch (error: any) { createDocumentCollectionMetric.error("Error create document", error); throw error; } + return dbResult; + } + + protected createDocumentDbValue(document: IDocument, isEphemeralContainer: boolean) { + const documentDbValue: IDocument & { ttl?: number } = { + ...document, + }; + if (isEphemeralContainer) { + documentDbValue.ttl = this.ephemeralDocumentTTLSec; + } + return documentDbValue; + } + + /** + * Basic validation for additionalQueryParams extensibility point. + * Override this method in derived classes for custom validation logic. + */ + protected validateAdditionalQueryParams(additionalQueryParams?: IAdditionalQueryParams): void { + if (!additionalQueryParams) { + return; + } + + // Log usage for security monitoring (in non-production or with privacy considerations) + const paramKeys = Object.keys(additionalQueryParams); + if (paramKeys.length > 0) { + Lumberjack.info("additionalQueryParams usage detected", { + paramCount: paramKeys.length, + // Only log keys, not values, for privacy + paramKeys: paramKeys.join(","), + }); + } + + this.validateObjectRecursive(additionalQueryParams, 0); + } + + /** + * Recursive validation helper + */ + private validateObjectRecursive(obj: IAdditionalQueryParams, currentDepth: number): void { + if (currentDepth > MAX_DEPTH) { + throw new NetworkError( + 400, + "additionalQueryParams: Maximum nesting depth exceeded", + false, + ); + } + + if (typeof obj !== "object" || obj === null) { + return; + } + + const keys = Object.keys(obj); + if (keys.length > MAX_OBJECT_SIZE) { + throw new NetworkError(400, "additionalQueryParams: Too many properties", false); + } + + for (const key of keys) { + if (key.length > MAX_KEY_LENGTH) { + throw new NetworkError(400, "additionalQueryParams: Key length exceeded", false); + } + + const value = obj[key]; + if (value === undefined) { + continue; + } + + if (typeof value === "string" && value.length > MAX_VALUE_LENGTH) { + throw new NetworkError(400, "additionalQueryParams: Value length exceeded", false); + } else if (Array.isArray(value)) { + if (value.length > MAX_OBJECT_SIZE) { + throw new NetworkError( + 400, + "additionalQueryParams: Array size exceeded", + false, + ); + } + for (const item of value) { + if (typeof item === "string" && item.length > MAX_VALUE_LENGTH) { + throw new NetworkError( + 400, + "additionalQueryParams: Array item length exceeded", + false, + ); + } else if (typeof item === "object") { + this.validateObjectRecursive(item, currentDepth + 1); + } + } + } else if (typeof value === "object" && value !== null) { + this.validateObjectRecursive(value, currentDepth + 1); + } + } + } + + protected async uploadSummary( + uploadManager: WholeSummaryUploadManager | SummaryTreeUploadManager, + fullTree: ISummaryTree, + gitManager: IGitManager, + documentId: string, + summaryTimeStr: string, + ) { + let summaryVersionId: string; + const handle = await uploadManager.writeSummaryTree( + fullTree /* summaryTree */, + "" /* parentHandle */, + "container" /* summaryType */, + 0 /* sequenceNumber */, + true /* initial */, + summaryTimeStr, + ); + + let summaryUploadMessage = `Tree reference: ${JSON.stringify(handle)}`; + + if (!this.enableWholeSummaryUpload) { + const commitParams: ICreateCommitParams = { + author: { + date: summaryTimeStr, + email: "dummy@microsoft.com", + name: "Routerlicious Service", + }, + message: "New document", + parents: [], + tree: handle, + }; + + const commit = await gitManager.createCommit(commitParams); + await gitManager.createRef(documentId, commit.sha); + // In the case of ShreddedSummary Upload, summary version is always the commit sha. + summaryVersionId = commit.sha; + summaryUploadMessage += ` - Commit sha: ${JSON.stringify(commit.sha)}`; + } else { + // In the case of WholeSummary Upload, summary tree handle is actually commit sha or version id. + summaryVersionId = handle; + } + return { summaryVersionId, summaryUploadMessage }; } public async getLatestVersion(tenantId: string, documentId: string): Promise { diff --git a/server/routerlicious/packages/services-telemetry/package.json b/server/routerlicious/packages/services-telemetry/package.json index b0e81dbc59b6..0bfbc2af2ba9 100644 --- a/server/routerlicious/packages/services-telemetry/package.json +++ b/server/routerlicious/packages/services-telemetry/package.json @@ -81,6 +81,12 @@ "broken": { "Enum_CommonProperties": { "backCompat": false + }, + "Enum_LumberEventName": { + "backCompat": false + }, + "Class_Lumber": { + "backCompat": false } }, "entrypoint": "public" diff --git a/server/routerlicious/packages/services-telemetry/src/lumberEventNames.ts b/server/routerlicious/packages/services-telemetry/src/lumberEventNames.ts index 6e9bda7f4e04..19bc16d360c0 100644 --- a/server/routerlicious/packages/services-telemetry/src/lumberEventNames.ts +++ b/server/routerlicious/packages/services-telemetry/src/lumberEventNames.ts @@ -56,6 +56,7 @@ export enum LumberEventName { ConnectDocumentOrdererConnection = "ConnectDocumentOrdererConnection", CreateDocumentUpdateDocumentCollection = "CreateDocumentUpdateDocumentCollection", CreateDocInitialSummaryWrite = "CreateDocInitialSummaryWrite", + CreateDocument = "CreateDocument", DisconnectDocument = "DisconnectDocument", DisconnectDocumentRetry = "DisconnectDocumentRetry", RiddlerFetchTenantKey = "RiddlerFetchTenantKey", diff --git a/server/routerlicious/packages/services-telemetry/src/resources.ts b/server/routerlicious/packages/services-telemetry/src/resources.ts index ee61262ddb42..7c164e30e65f 100644 --- a/server/routerlicious/packages/services-telemetry/src/resources.ts +++ b/server/routerlicious/packages/services-telemetry/src/resources.ts @@ -131,6 +131,7 @@ export enum CommonProperties { isGlobalDb = "isGlobalDb", internalErrorCode = "internalErrorCode", callingServiceName = "callingServiceName", + uploadAsEphemeralContainer = "uploadAsEphemeralContainer", } /** diff --git a/server/routerlicious/packages/services-telemetry/src/test/types/validateServerServicesTelemetryPrevious.generated.ts b/server/routerlicious/packages/services-telemetry/src/test/types/validateServerServicesTelemetryPrevious.generated.ts index 38f2415a5c21..0ce271103136 100644 --- a/server/routerlicious/packages/services-telemetry/src/test/types/validateServerServicesTelemetryPrevious.generated.ts +++ b/server/routerlicious/packages/services-telemetry/src/test/types/validateServerServicesTelemetryPrevious.generated.ts @@ -103,6 +103,7 @@ declare type old_as_current_for_Class_Lumber = requireAssignableTo, TypeOnly> /* @@ -428,6 +429,7 @@ declare type old_as_current_for_Enum_LumberEventName = requireAssignableTo, TypeOnly> /* diff --git a/server/routerlicious/packages/services-utils/src/asyncContext.ts b/server/routerlicious/packages/services-utils/src/asyncContext.ts index f027ec5ecaca..b93cbd4a3c00 100644 --- a/server/routerlicious/packages/services-utils/src/asyncContext.ts +++ b/server/routerlicious/packages/services-utils/src/asyncContext.ts @@ -27,7 +27,7 @@ export class AsyncLocalStorageContextProvider { * If properties are a key-value record, new entries will be appended to the existing record. * Otherwise, the old context will be overwritten with the new context. */ - public bindContext(props: T, callback: () => void): void { + public bindContext(props: T, callback: () => TBindType): TBindType { // Extend existing properties if props are a key-value record. // Otherwise, overwrite existing props with new props. const existingProps = this.getContext(); @@ -36,7 +36,7 @@ export class AsyncLocalStorageContextProvider { ? { ...existingProps, ...props } : props; // Anything within callback context will have access to properties. - this.asyncLocalStorage.run(newProperties, () => callback()); + return this.asyncLocalStorage.run(newProperties, () => callback()); } /** diff --git a/server/routerlicious/packages/services/src/documentManager.ts b/server/routerlicious/packages/services/src/documentManager.ts index 618ca8bee7e1..710a9194eb33 100644 --- a/server/routerlicious/packages/services/src/documentManager.ts +++ b/server/routerlicious/packages/services/src/documentManager.ts @@ -71,8 +71,7 @@ export class DocumentManager implements IDocumentManager { "Falling back to database after attempting to read cached static document data, because the DocumentManager cache is undefined.", getLumberBaseProperties(documentId, tenantId), ); - const document = (await this.readDocument(tenantId, documentId)) ?? undefined; - return document as IDocumentStaticProperties | undefined; + return this.getDocumentStaticProperties(tenantId, documentId); } // Retrieve cached static document props @@ -86,15 +85,7 @@ export class DocumentManager implements IDocumentManager { "Falling back to database after attempting to read cached static document data.", getLumberBaseProperties(documentId, tenantId), ); - const document = await this.readDocument(tenantId, documentId); - if (!document) { - Lumberjack.warning( - "Fallback to database failed, document not found.", - getLumberBaseProperties(documentId, tenantId), - ); - return undefined; - } - return DocumentManager.getStaticPropsFromDoc(document); + return this.getDocumentStaticProperties(tenantId, documentId); } // Return the static data, parsed into a JSON object @@ -123,6 +114,21 @@ export class DocumentManager implements IDocumentManager { await this.documentStaticDataCache.delete(staticPropsKey); } + private async getDocumentStaticProperties( + tenantId: string, + documentId: string, + ): Promise { + const document = await this.readDocument(tenantId, documentId); + if (!document) { + Lumberjack.warning( + "Fallback to database failed, document not found.", + getLumberBaseProperties(documentId, tenantId), + ); + return undefined; + } + return DocumentManager.getStaticPropsFromDoc(document); + } + private async getBasicRestWrapper(tenantId: string, documentId: string) { const scopes = [ScopeType.DocRead]; const accessToken = await this.tenantManager.signToken(tenantId, documentId, scopes); diff --git a/server/routerlicious/packages/services/src/tenant.ts b/server/routerlicious/packages/services/src/tenant.ts index 821765971799..e66ebbae4eda 100644 --- a/server/routerlicious/packages/services/src/tenant.ts +++ b/server/routerlicious/packages/services/src/tenant.ts @@ -14,6 +14,8 @@ import { parseToken, isNetworkError, NetworkError, + type GitManagerConfigDecorator, + type IGitManagerConfig, } from "@fluidframework/server-services-client"; import * as core from "@fluidframework/server-services-core"; import type { IInvalidTokenError } from "@fluidframework/server-services-core"; @@ -153,10 +155,18 @@ export class TenantManager implements core.ITenantManager, core.ITenantConfigMan tenantId: string, documentId: string, includeDisabledTenant = false, + configDecorator?: GitManagerConfigDecorator, ): Promise { const [details, gitManager] = await Promise.all([ this.getTenantConfig(tenantId, includeDisabledTenant), - this.getTenantGitManager(tenantId, documentId, undefined, includeDisabledTenant), + this.getTenantGitManager( + tenantId, + documentId, + undefined, + includeDisabledTenant, + false, + configDecorator, + ), ]); const tenant = new Tenant(details, gitManager); @@ -164,12 +174,80 @@ export class TenantManager implements core.ITenantManager, core.ITenantConfigMan return tenant; } + /** + * Gets a GitManager instance for a specific tenant and document. + * + * @param tenantId - The tenant identifier + * @param documentId - The document identifier + * @param storageName - Optional storage name for routing + * @param includeDisabledTenant - Whether to include disabled tenants + * @param isEphemeralContainer - Whether this is for an ephemeral container + * @param configDecorator - Optional decorator to customize the GitManager configuration + * (Security: Critical functions are protected from override) + * + * @example Basic usage: + * ```typescript + * const gitManager = await tenantManager.getTenantGitManager(tenantId, documentId); + * ``` + * + * @example With custom headers: + * ```typescript + * const gitManager = await tenantManager.getTenantGitManager( + * tenantId, + * documentId, + * undefined, + * false, + * false, + * GitManagerConfigDecorators.withCustom({ + * defaultHeaders: { + * 'X-Custom-Header': 'custom-value', + * 'X-Request-ID': requestId, + * } + * }) + * ); + * ``` + * + * @example With custom metrics and query params: + * ```typescript + * const gitManager = await tenantManager.getTenantGitManager( + * tenantId, documentId, undefined, false, false, + * GitManagerConfigDecorators.withCustom({ + * defaultQueryString: { version: '2.0' }, + * logHttpMetrics: (props) => { + * console.log('Custom HTTP metrics:', props); + * // Your custom logging logic here + * }, + * }) + * ); + * ``` + * + * @example Custom decorator for specific tenant needs: + * ```typescript + * const customDecorator = GitManagerConfigDecorators.withCustom((config, context) => ({ + * defaultHeaders: { + * 'X-Tenant-ID': context.tenantId, // ✅ Safe custom header + * 'X-Document-ID': context.documentId, // ✅ Safe custom header + * 'X-Storage-Name': context.storageName || 'default', // ✅ Safe custom header + * }, + * maxBodyLength: context.isEphemeralContainer ? 100 * 1024 : 1000 * 1024, // ✅ Safe override + * // refreshTokenIfNeeded: customRefresh, // ❌ This would be ignored for security + * })); + * + * // Or even simpler for static overrides: + * const simpleDecorator = GitManagerConfigDecorators.withCustom({ + * maxBodyLength: 2000 * 1024, // ✅ Safe limit + * defaultQueryString: { version: '2.0', source: 'custom' }, // ✅ Safe params + * logHttpMetrics: customMetricsLogger, // ✅ Safe (with validation) + * }); + * ``` + */ public async getTenantGitManager( tenantId: string, documentId: string, storageName?: string, includeDisabledTenant = false, isEphemeralContainer = false, + configDecorator?: GitManagerConfigDecorator, ): Promise { const lumberProperties = { ...getLumberBaseProperties(documentId, tenantId), @@ -182,6 +260,9 @@ export class TenantManager implements core.ITenantManager, core.ITenantConfigMan lumberProperties /* telemetryProperties */, ); + const baseUrl = `${this.internalHistorianUrl}/repos/${encodeURIComponent(tenantId)}`; + + // Create default configuration const defaultQueryString = {}; const getDefaultHeaders = () => { const credentials: ICredentials = { @@ -240,22 +321,45 @@ export class TenantManager implements core.ITenantManager, core.ITenantConfigMan } } }; - const defaultHeaders = getDefaultHeaders(); - const baseUrl = `${this.internalHistorianUrl}/repos/${encodeURIComponent(tenantId)}`; - const tenantRestWrapper = new BasicRestWrapper( - baseUrl, + + // Create default configuration + let config: IGitManagerConfig = { defaultQueryString, - undefined, - undefined, - defaultHeaders, - undefined, - undefined, + defaultHeaders: getDefaultHeaders(), getDefaultHeaders, - () => getGlobalTelemetryContext().getProperties().correlationId, - () => getGlobalTelemetryContext().getProperties(), refreshTokenIfNeeded, + getCorrelationId: () => getGlobalTelemetryContext().getProperties().correlationId, + getTelemetryProperties: () => getGlobalTelemetryContext().getProperties(), logHttpMetrics, - () => getGlobalTelemetryContext().getProperties().serviceName ?? "" /* serviceName */, + getServiceName: () => getGlobalTelemetryContext().getProperties().serviceName ?? "", + }; + + // Apply decorator if provided + if (configDecorator) { + config = configDecorator(config, { + tenantId, + documentId, + storageName, + isEphemeralContainer, + accessToken, + baseUrl, + }); + } + + const tenantRestWrapper = new BasicRestWrapper( + baseUrl, + config.defaultQueryString, + config.maxBodyLength, + config.maxContentLength, + config.defaultHeaders, + undefined, + config.refreshDefaultQueryString, + config.refreshDefaultHeaders ?? config.getDefaultHeaders, + config.getCorrelationId, + config.getTelemetryProperties, + config.refreshTokenIfNeeded, + config.logHttpMetrics, + config.getServiceName, ); const historian = new Historian(baseUrl, true, false, tenantRestWrapper); const gitManager = new GitManager(historian); From f4ec6235b83908b64bdba2eda210d8565bba217a Mon Sep 17 00:00:00 2001 From: zhangxin511 Date: Wed, 8 Oct 2025 10:33:29 -0400 Subject: [PATCH 2/3] Update server/routerlicious/packages/lambdas/src/scribe/interfaces.ts Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- server/routerlicious/packages/lambdas/src/scribe/interfaces.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts b/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts index f05c93c7ff94..3c1aa2107141 100644 --- a/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts +++ b/server/routerlicious/packages/lambdas/src/scribe/interfaces.ts @@ -99,8 +99,8 @@ export interface ICheckpointManager { export interface ISummaryWriterFactory { create( - documentId: string, tenantId: string, + documentId: string, gitManager: IGitManager, deltaManager: IDeltaService, messageCollection: ICollection, From d0f78115b6d2fa4aab75e5c696997a0c2921cbf7 Mon Sep 17 00:00:00 2001 From: Xin Zhang Date: Wed, 8 Oct 2025 11:01:27 -0400 Subject: [PATCH 3/3] Clean up old code --- server/routerlicious/packages/services-shared/src/storage.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/server/routerlicious/packages/services-shared/src/storage.ts b/server/routerlicious/packages/services-shared/src/storage.ts index fbba6c98db78..4b7cfa253f84 100644 --- a/server/routerlicious/packages/services-shared/src/storage.ts +++ b/server/routerlicious/packages/services-shared/src/storage.ts @@ -246,7 +246,6 @@ export class DocumentStorage implements IDocumentStorage { tenantId, storageName, isEphemeralContainer, - additionalQueryParams, ); createDocTrace.stampStage(DocCreationStage.DocCreationCompleted); return dbResult;