From b536a733c6115ea11c426c8ba9176e35b052195b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 29 Sep 2025 20:04:37 +0300 Subject: [PATCH 01/24] expose joinGraph from transformQueryToCanUseForm() --- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 106319d1ed55f..e65e77dd4445f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -510,6 +510,7 @@ export class PreAggregations { allValuesEq1(filterDimensionsSingleValueEqual) ? new Set(filterDimensionsSingleValueEqual?.keys()) : null; return { + joinGraph: query.join, sortedDimensions, sortedTimeDimensions, timeDimensions, From e79af2543494a64bfe241d410ac858977ebc656e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 15:43:45 +0300 Subject: [PATCH 02/24] get rid of ramda in favor of simple js --- .../src/adapter/PreAggregations.ts | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index e65e77dd4445f..73238c63b354d 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -733,14 +733,10 @@ export class PreAggregations { // no connections in the joinTree between cubes from different datasources const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions; - const dimensionsMatch = (dimensions, doBackAlias) => R.all( - d => ( - doBackAlias ? - backAlias(dimsToMatch) : - (dimsToMatch) - ).indexOf(d) !== -1, - dimensions - ); + const dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are // no connections in the joinTree between cubes from different datasources From 97c62dc10091a40723af977a6be98ce8c5cfaec6 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 15:55:24 +0300 Subject: [PATCH 03/24] preparing dimensionsMatch() --- .../src/adapter/PreAggregations.ts | 33 ++++++++++++------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 73238c63b354d..5b63b78ec0dd3 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -729,18 +729,29 @@ export class PreAggregations { } } - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - const dimsToMatch = references.rollups.length > 0 ? references.dimensions : references.fullNameDimensions; - - const dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; - return dimensions.every(d => target.includes(d)); - }; + let dimsToMatch: string[]; + let timeDimsToMatch: PreAggregationTimeDimensionReference[]; + let dimensionsMatch: (dimensions: string[], doBackAlias: boolean) => boolean; + + if (references.rollups.length > 0) { + // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are + // no connections in the joinTree between cubes from different datasources + dimsToMatch = references.dimensions; + timeDimsToMatch = references.timeDimensions; + + dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; + } else { + dimsToMatch = references.fullNameDimensions; + timeDimsToMatch = references.fullNameTimeDimensions; - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - const timeDimsToMatch = references.rollups.length > 0 ? references.timeDimensions : references.fullNameTimeDimensions; + dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; + } const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( From 27ac5bc734617a33e214d31f5f33556e2d03ee88 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 17:26:47 +0300 Subject: [PATCH 04/24] fix(schema-compiler): Fix pre-agg matching for 'rollupJoin' / 'rollupLambda' pre-aggregations --- .../src/adapter/PreAggregations.ts | 36 +++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 5b63b78ec0dd3..d37124ec60100 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -509,8 +509,18 @@ export class PreAggregations { filterDimensionsSingleValueEqual = allValuesEq1(filterDimensionsSingleValueEqual) ? new Set(filterDimensionsSingleValueEqual?.keys()) : null; + // Build reverse query joins map, which is used for + // rollupLambda and rollupJoin pre-aggs matching later + const joinsMap: Record = {}; + if (query.join) { + for (const j of query.join.joins) { + joinsMap[j.to] = j.from; + } + } + return { - joinGraph: query.join, + joinGraphRoot: query.join?.root, + joinsMap, sortedDimensions, sortedTimeDimensions, timeDimensions, @@ -736,11 +746,33 @@ export class PreAggregations { if (references.rollups.length > 0) { // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are // no connections in the joinTree between cubes from different datasources + // but joinGraph of the query has all the connections, necessary for serving the query, + // so we use this information to complete the full paths of members from the root of the query + // up to the pre-agg cube. dimsToMatch = references.dimensions; timeDimsToMatch = references.timeDimensions; + const buildPath = (cube: string): string[] => { + const path = [cube]; + const parentMap = transformedQuery.joinsMap; + while (parentMap[cube]) { + cube = parentMap[cube]; + path.push(cube); + } + return path.reverse(); + }; + dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + let target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + target = target.map(dim => { + const [cube, field] = dim.split('.'); + if (cube === transformedQuery.joinGraphRoot) { + return dim; + } + const path = buildPath(cube); + return `${path.join('.')}.${field}`; + }); + return dimensions.every(d => target.includes(d)); }; } else { From 961707a34958c2ff2e470a1d0716329bfeed7446 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 30 Sep 2025 17:44:00 +0300 Subject: [PATCH 05/24] add tests --- .../postgres/pre-aggregations.test.ts | 110 +++++++++++++++++- 1 file changed, 109 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 43fd4b6ced43b..7499d7ad773d9 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -599,7 +599,83 @@ describe('PreAggregations', () => { } ] }); - `); + + cube('cube_1', { + sql: \`SELECT 1 as id, 'dim_1' as dim_1\`, + + joins: { + cube_2: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_1} = \${cube_2.dim_1}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_1: { + sql: 'dim_1', + type: 'string' + }, + }, + + pre_aggregations: { + aaa: { + dimensions: [ + dim_1 + ] + }, + rollupJoin: { + type: 'rollupJoin', + dimensions: [ + dim_1, + cube_2.dim_1, + cube_2.dim_2 // XXX + ], + rollups: [ + aaa, + cube_2.bbb + ] + } + } + }); + + cube('cube_2', { + sql: \`SELECT 2 as id, 'dim_1' as dim_1, 'dim_2' as dim_2\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_1: { + sql: 'dim_1', + type: 'string' + }, + + dim_2: { + sql: 'dim_2', + type: 'string' + }, + }, + + pre_aggregations: { + bbb: { + dimensions: [ + dim_1, + dim_2, + ] + } + } + }); + + `); it('simple pre-aggregation', async () => { await compiler.compile(); @@ -2773,4 +2849,36 @@ describe('PreAggregations', () => { expect(loadSql[0]).not.toMatch(/GROUP BY/); expect(loadSql[0]).toMatch(/THEN 1 END `real_time_lambda_visitors__count`/); }); + + it('rollupJoin pre-aggregation', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_1.dim_1', 'cube_2.dim_2'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(2); + const aaa = preAggregationsDescription.find(p => p.preAggregationId === 'cube_1.aaa'); + const bbb = preAggregationsDescription.find(p => p.preAggregationId === 'cube_2.bbb'); + expect(aaa).toBeDefined(); + expect(bbb).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoin'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_1__dim_1: 'dim_1', + cube_2__dim_2: 'dim_2', + }] + ); + }); + }); }); From ed56c9dd7e2b8ec0e2e564bc539c1354d7ea85ed Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 14:42:07 +0300 Subject: [PATCH 06/24] add test for 3-cube rollupJoin pre-agg --- .../postgres/pre-aggregations.test.ts | 149 ++++++++++++++++++ 1 file changed, 149 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 7499d7ad773d9..72e10fb6a71fa 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -675,6 +675,120 @@ describe('PreAggregations', () => { } }); + cube('cube_x', { + sql: \`SELECT 1 as id, 'dim_x' as dim_x\`, + + joins: { + cube_y: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_x} = \${cube_y.dim_x}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_x: { + sql: 'dim_x', + type: 'string' + }, + }, + + pre_aggregations: { + xxx: { + dimensions: [ + dim_x + ] + }, + rollupJoinThreeCubes: { + type: 'rollupJoin', + dimensions: [ + dim_x, + cube_y.dim_y, + cube_z.dim_z + ], + rollups: [ + xxx, + cube_y.yyy, + cube_z.zzz + ] + } + } + }); + + cube('cube_y', { + sql: \`SELECT 2 as id, 'dim_x' as dim_x, 'dim_y' as dim_y\`, + + joins: { + cube_z: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_y} = \${cube_z.dim_y}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_x: { + sql: 'dim_x', + type: 'string' + }, + + dim_y: { + sql: 'dim_y', + type: 'string' + }, + }, + + pre_aggregations: { + yyy: { + dimensions: [ + dim_x, + dim_y, + ] + } + } + }); + + cube('cube_z', { + sql: \`SELECT 3 as id, 'dim_y' as dim_y, 'dim_z' as dim_z\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_y: { + sql: 'dim_y', + type: 'string' + }, + + dim_z: { + sql: 'dim_z', + type: 'string' + }, + }, + + pre_aggregations: { + zzz: { + dimensions: [ + dim_y, + dim_z, + ] + } + } + }); + `); it('simple pre-aggregation', async () => { @@ -2881,4 +2995,39 @@ describe('PreAggregations', () => { ); }); }); + + it('rollupJoin pre-aggregation with three cubes', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_x.dim_x', 'cube_y.dim_y', 'cube_z.dim_z'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(3); + const xxx = preAggregationsDescription.find(p => p.preAggregationId === 'cube_x.xxx'); + const yyy = preAggregationsDescription.find(p => p.preAggregationId === 'cube_y.yyy'); + const zzz = preAggregationsDescription.find(p => p.preAggregationId === 'cube_z.zzz'); + expect(xxx).toBeDefined(); + expect(yyy).toBeDefined(); + expect(zzz).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinThreeCubes'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_x__dim_x: 'dim_x', + cube_y__dim_y: 'dim_y', + cube_z__dim_z: 'dim_z', + }] + ); + }); + }); }); From a1361be5c927c9377887a9466b12c8b66b7ba845 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 18:47:33 +0300 Subject: [PATCH 07/24] use rollupsReferences for 'rollupJoin' / 'rollupLambda' pre-agg matching --- .../src/adapter/PreAggregations.ts | 27 +++++++++++++++---- .../src/compiler/CubeEvaluator.ts | 2 ++ 2 files changed, 24 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index d37124ec60100..c259412a43945 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -749,8 +749,15 @@ export class PreAggregations { // but joinGraph of the query has all the connections, necessary for serving the query, // so we use this information to complete the full paths of members from the root of the query // up to the pre-agg cube. - dimsToMatch = references.dimensions; - timeDimsToMatch = references.timeDimensions; + // We use references from the underlying pre-aggregations, filtered with members existing in the root + // pre-aggregation itself. + + dimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameDimensions) + .filter(d => references.dimensions.some(rd => d.endsWith(rd))); + timeDimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameTimeDimensions) + .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); const buildPath = (cube: string): string[] => { const path = [cube]; @@ -765,12 +772,12 @@ export class PreAggregations { dimensionsMatch = (dimensions, doBackAlias) => { let target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; target = target.map(dim => { - const [cube, field] = dim.split('.'); + const [cube, ...restPath] = dim.split('.'); if (cube === transformedQuery.joinGraphRoot) { return dim; } const path = buildPath(cube); - return `${path.join('.')}.${field}`; + return `${path.join('.')}.${restPath.join('.')}`; }); return dimensions.every(d => target.includes(d)); @@ -1084,7 +1091,9 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - canUsePreAggregation: canUsePreAggregation(references), + // For rollupJoin and rollupLambda we need to pass references of the underlying rollups + // to canUsePreAggregation fn, which are collected later; + canUsePreAggregation: preAggregation.type === 'rollup' ? canUsePreAggregation(references) : false, references, preAggregationId: `${cube}.${preAggregationName}` }; @@ -1102,8 +1111,12 @@ export class PreAggregations { ); } ); + preAggregationsToJoin.forEach(preAgg => { + references.rollupsReferences.push(preAgg.references); + }); return { ...preAggObj, + canUsePreAggregation: canUsePreAggregation(references), preAggregationsToJoin, rollupJoin: this.buildRollupJoin(preAggObj, preAggregationsToJoin) }; @@ -1150,8 +1163,12 @@ export class PreAggregations { PreAggregations.memberNameMismatchValidation(preAggObj, referencedPreAggregation, 'dimensions'); PreAggregations.memberNameMismatchValidation(preAggObj, referencedPreAggregation, 'timeDimensions'); }); + referencedPreAggregations.forEach(preAgg => { + references.rollupsReferences.push(preAgg.references); + }); return { ...preAggObj, + canUsePreAggregation: canUsePreAggregation(references), referencedPreAggregations, }; } else { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index e540183d3d9a8..f89ddad1da0db 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -95,6 +95,7 @@ export type PreAggregationReferences = { timeDimensions: Array, fullNameTimeDimensions: Array, rollups: Array, + rollupsReferences: Array, multipliedMeasures?: Array, joinTree?: FinishedJoinTree; }; @@ -891,6 +892,7 @@ export class CubeEvaluator extends CubeSymbols { fullNameDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() fullNameMeasures: [], // May be filled in PreAggregations.evaluateAllReferences() fullNameTimeDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() + rollupsReferences: [], // May be filled in PreAggregations.evaluateAllReferences() }; } } From f3774a843660125a06b421f3f0880d4b8cf60b05 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 18:47:52 +0300 Subject: [PATCH 08/24] fix old tests with new required fields --- .../test/unit/pre-agg-by-filter-match.test.ts | 10 +++++++--- .../test/unit/pre-agg-time-dim-match.test.ts | 10 +++++++--- .../test/unit/RefreshScheduler.test.ts | 1 + 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts index 004fb1de3c4f4..7253ca3c35720 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts @@ -59,9 +59,13 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + fullNameDimensions: testPreAgg.segments ? testPreAgg.dimensions.concat(testPreAgg.segments) : testPreAgg.dimensions, + fullNameMeasures: testPreAgg.measures, + fullNameTimeDimensions: [{ + dimension: testPreAgg.timeDimension, + granularity: testPreAgg.granularity, + }], + rollupsReferences: [], }; await compiler.compile(); diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index bea487909c743..56d294701ba6c 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -69,9 +69,13 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], + fullNameDimensions: testPreAgg.dimensions, + fullNameMeasures: testPreAgg.measures, + fullNameTimeDimensions: [{ + dimension: testPreAgg.timeDimension, + granularity: testPreAgg.granularity, + }], + rollupsReferences: [], }; await compiler.compile(); diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 1ba8b3622a166..370dd90d82742 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -665,6 +665,7 @@ describe('Refresh Scheduler', () => { measures: ['Foo.count'], timeDimensions: [{ dimension: 'Foo.time', granularity: 'hour' }], rollups: [], + rollupsReferences: [], fullNameDimensions: [], fullNameMeasures: [], fullNameTimeDimensions: [], From 55a34191634288a5b1c22637bcf5b1826aa620da Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 18:48:24 +0300 Subject: [PATCH 09/24] more tests --- .../postgres/pre-aggregations.test.ts | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 72e10fb6a71fa..5503cbfc5476c 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -789,6 +789,142 @@ describe('PreAggregations', () => { } }); + cube('cube_a', { + sql: \`SELECT 1 as id, 'dim_a' as dim_a\`, + + joins: { + cube_b: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_b.dim_a}\` + }, + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_a} = \${cube_c.dim_a}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + aaa_rollup: { + dimensions: [ + dim_a + ] + }, + rollupJoinAB: { + type: 'rollupJoin', + dimensions: [ + dim_a, + cube_b.dim_b, + cube_c.dim_c + ], + rollups: [ + aaa_rollup, + cube_b.bbb_rollup + ] + } + } + }); + + cube('cube_b', { + sql: \`SELECT 2 as id, 'dim_a' as dim_a, 'dim_b' as dim_b\`, + + joins: { + cube_c: { + relationship: 'many_to_one', + sql: \`\${CUBE.dim_b} = \${cube_c.dim_b}\` + } + }, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + }, + + pre_aggregations: { + bbb_rollup: { + dimensions: [ + dim_a, + dim_b, + cube_c.dim_c + ] + } + } + }); + + cube('cube_c', { + sql: \`SELECT 3 as id, 'dim_a' as dim_a, 'dim_b' as dim_b, 'dim_c' as dim_c\`, + + dimensions: { + id: { + sql: 'id', + type: 'string', + primary_key: true + }, + + dim_a: { + sql: 'dim_a', + type: 'string' + }, + + dim_b: { + sql: 'dim_b', + type: 'string' + }, + + dim_c: { + sql: 'dim_c', + type: 'string' + }, + } + }); + + view('view_abc', { + cubes: [ + { + join_path: cube_a, + includes: ['dim_a'] + }, + { + join_path: cube_a.cube_b, + includes: ['dim_b'] + }, + { + join_path: cube_a.cube_b.cube_c, + includes: ['dim_c'] + } + ] + }); + `); it('simple pre-aggregation', async () => { @@ -3030,4 +3166,65 @@ describe('PreAggregations', () => { ); }); }); + + it('rollupJoin pre-aggregation with nested joins via view (A->B->C)', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['view_abc.dim_a', 'view_abc.dim_b', 'view_abc.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(2); + const aaa = preAggregationsDescription.find(p => p.preAggregationId === 'cube_a.aaa_rollup'); + const bbb = preAggregationsDescription.find(p => p.preAggregationId === 'cube_b.bbb_rollup'); + expect(aaa).toBeDefined(); + expect(bbb).toBeDefined(); + + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinAB'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + view_abc__dim_a: 'dim_a', + view_abc__dim_b: 'dim_b', + view_abc__dim_c: 'dim_c', + }] + ); + }); + }); + + it('rollupJoin pre-aggregation with nested joins via cube (A->B->C)', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_a.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(0); + + expect(query.preAggregations?.preAggregationForQuery).toBeUndefined(); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_a__dim_a: 'dim_a', + cube_b__dim_b: 'dim_b', + cube_c__dim_c: 'dim_c', + }] + ); + }); + }); }); From 3a0bcdbf0ea9138fbec8c27a5244bcdf5d85b3da Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 20:54:42 +0300 Subject: [PATCH 10/24] implement pre-agg matching using pre-agg join subgraphs --- .../src/adapter/PreAggregations.ts | 106 ++++++++++++------ 1 file changed, 71 insertions(+), 35 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index c259412a43945..0285fc9be75af 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -639,6 +639,16 @@ export class PreAggregations { transformedQuery.allBackAliasMembers[r] || r )); + const buildPath = (cube: string): string[] => { + const path = [cube]; + const parentMap = transformedQuery.joinsMap; + while (parentMap[cube]) { + cube = parentMap[cube]; + path.push(cube); + } + return path.reverse(); + }; + /** * Determine whether pre-aggregation can be used or not. */ @@ -647,8 +657,32 @@ export class PreAggregations { const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - const backAliasMeasures = backAlias(references.measures); - const backAliasDimensions = backAlias(references.dimensions); + + let dimsToMatch: string[]; + let measToMatch: string[]; + + if (references.rollups.length > 0) { + // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are + // no connections in the joinTree between cubes from different datasources + // but joinGraph of the query has all the connections, necessary for serving the query, + // so we use this information to complete the full paths of members from the root of the query + // up to the pre-agg cube. + // We use references from the underlying pre-aggregations, filtered with members existing in the root + // pre-aggregation itself. + + dimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameDimensions) + .filter(d => references.dimensions.some(rd => d.endsWith(rd))); + measToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameMeasures) + .filter(m => references.measures.some(rm => m.endsWith(rm))); + } else { + dimsToMatch = references.fullNameDimensions; + measToMatch = references.fullNameMeasures; + } + + const backAliasMeasures = backAlias(measToMatch); + const backAliasDimensions = backAlias(dimsToMatch); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -741,7 +775,6 @@ export class PreAggregations { let dimsToMatch: string[]; let timeDimsToMatch: PreAggregationTimeDimensionReference[]; - let dimensionsMatch: (dimensions: string[], doBackAlias: boolean) => boolean; if (references.rollups.length > 0) { // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are @@ -754,44 +787,28 @@ export class PreAggregations { dimsToMatch = references.rollupsReferences .flatMap(rolRef => rolRef.fullNameDimensions) - .filter(d => references.dimensions.some(rd => d.endsWith(rd))); - timeDimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameTimeDimensions) - .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); - - const buildPath = (cube: string): string[] => { - const path = [cube]; - const parentMap = transformedQuery.joinsMap; - while (parentMap[cube]) { - cube = parentMap[cube]; - path.push(cube); - } - return path.reverse(); - }; - - dimensionsMatch = (dimensions, doBackAlias) => { - let target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; - target = target.map(dim => { - const [cube, ...restPath] = dim.split('.'); + .filter(d => references.dimensions.some(rd => d.endsWith(rd))) + .map(d => { + const [cube, ...restPath] = d.split('.'); if (cube === transformedQuery.joinGraphRoot) { - return dim; + return d; } const path = buildPath(cube); return `${path.join('.')}.${restPath.join('.')}`; }); - - return dimensions.every(d => target.includes(d)); - }; + timeDimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameTimeDimensions) + .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); } else { dimsToMatch = references.fullNameDimensions; timeDimsToMatch = references.fullNameTimeDimensions; - - dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; - return dimensions.every(d => target.includes(d)); - }; } + const dimensionsMatch = (dimensions, doBackAlias) => { + const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + return dimensions.every(d => target.includes(d)); + }; + const timeDimensionsMatch = (timeDimensionsList, doBackAlias) => R.allPass( timeDimensionsList.map( tds => R.anyPass(tds.map((td: [string, string]) => { @@ -1005,10 +1022,28 @@ export class PreAggregations { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { + // It's important to build join graph not only using the pre-agg members, but also + // taking into account all explicit underlying rollup pre-aggregation joins, because + // otherwise the built join tree might differ from the actual pre-aggregation. + const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map(r => { + if (!r.joinTree) { + return []; + } + + const hints: (string | string[])[] = [r.joinTree.root]; + + for (const j of r.joinTree.joins) { + hints.push([j.from, j.to]); + } + + return hints; + }).flat(); const targetJoins = this.resolveJoinMembers( - // TODO join hints? - this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj)) + this.query.joinGraph.buildJoin( + preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) + ) ); + // const targetJoins = this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj))); const existingJoins = R.unnest(preAggObjsToJoin.map( // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) @@ -1114,11 +1149,12 @@ export class PreAggregations { preAggregationsToJoin.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); }); + const canUsePreAggregationResult = canUsePreAggregation(references); return { ...preAggObj, - canUsePreAggregation: canUsePreAggregation(references), + canUsePreAggregation: canUsePreAggregationResult, preAggregationsToJoin, - rollupJoin: this.buildRollupJoin(preAggObj, preAggregationsToJoin) + rollupJoin: canUsePreAggregationResult ? this.buildRollupJoin(preAggObj, preAggregationsToJoin) : null, }; } else if (preAggregation.type === 'rollupLambda') { // TODO evaluation optimizations. Should be cached or moved to compile time. From 58338d286683b0131bdec5120107b4ed0351759d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 21:25:55 +0300 Subject: [PATCH 11/24] fix incorrect cache for pre-aggs --- .../src/adapter/PreAggregations.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 0285fc9be75af..b24094da576b0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1414,7 +1414,15 @@ export class PreAggregations { if (!preAggregationName) { return evaluateReferences(); } - return this.query.cacheValue(['evaluateAllReferences', cube, preAggregationName], evaluateReferences); + + // Using [cube, preAggregationName] alone as cache keys isn’t reliable, + // as different queries can build distinct join graphs during pre-aggregation matching. + // Because the matching logic compares join subgraphs — particularly for 'rollupJoin' and 'rollupLambda' + // pre-aggregations — relying on such keys may cause incorrect results. + return this.query.cacheValue( + ['evaluateAllReferences', cube, preAggregationName, JSON.stringify(this.query.join)], + evaluateReferences + ); } public originalSqlPreAggregationTable(preAggregationDescription: PreAggregationForCube): string { From 239d11c18023c71653509a42ed01f0e92ff61c59 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Mon, 20 Oct 2025 23:03:36 +0300 Subject: [PATCH 12/24] fix canUsePreAggregationNotAdditive --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index b24094da576b0..35332274af29f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -653,13 +653,13 @@ export class PreAggregations { * Determine whether pre-aggregation can be used or not. */ const canUsePreAggregationNotAdditive: CanUsePreAggregationFn = (references: PreAggregationReferences): boolean => { - const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); const qryTimeDimensions = references.allowNonStrictDateRangeMatch ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; let dimsToMatch: string[]; let measToMatch: string[]; + let timeDimsToMatch: PreAggregationTimeDimensionReference[]; if (references.rollups.length > 0) { // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are @@ -673,14 +673,19 @@ export class PreAggregations { dimsToMatch = references.rollupsReferences .flatMap(rolRef => rolRef.fullNameDimensions) .filter(d => references.dimensions.some(rd => d.endsWith(rd))); + timeDimsToMatch = references.rollupsReferences + .flatMap(rolRef => rolRef.fullNameTimeDimensions) + .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); measToMatch = references.rollupsReferences .flatMap(rolRef => rolRef.fullNameMeasures) .filter(m => references.measures.some(rm => m.endsWith(rm))); } else { dimsToMatch = references.fullNameDimensions; + timeDimsToMatch = references.fullNameTimeDimensions; measToMatch = references.fullNameMeasures; } + const refTimeDimensions = backAlias(sortTimeDimensions(timeDimsToMatch)); const backAliasMeasures = backAlias(measToMatch); const backAliasDimensions = backAlias(dimsToMatch); return (( @@ -1043,7 +1048,6 @@ export class PreAggregations { preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) ) ); - // const targetJoins = this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(preAggObj))); const existingJoins = R.unnest(preAggObjsToJoin.map( // TODO join hints? p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) From 75e9cdd30e14939c22d4f5f585a90b457c930d3b Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Tue, 21 Oct 2025 19:39:05 +0300 Subject: [PATCH 13/24] skip test for tesseract --- .../postgres/pre-aggregations.test.ts | 51 +++++++++++-------- 1 file changed, 29 insertions(+), 22 deletions(-) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 5503cbfc5476c..66148cea4e02c 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -3200,31 +3200,38 @@ describe('PreAggregations', () => { }); }); - it('rollupJoin pre-aggregation with nested joins via cube (A->B->C)', async () => { - await compiler.compile(); - - const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { - dimensions: ['cube_a.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], - timezone: 'America/Los_Angeles', - preAggregationsSchema: '' + if (getEnv('nativeSqlPlanner')) { + it.skip('FIXME(tesseract): rollupJoin pre-aggregation with nested joins via cube (A->B->C)', () => { + // Need to investigate tesseract internals of how pre-aggs members are resolved and how + // rollups are used to construct rollupJoins. }); + } else { + it('rollupJoin pre-aggregation with nested joins via cube (A->B->C)', async () => { + await compiler.compile(); - const queryAndParams = query.buildSqlAndParams(); - console.log(queryAndParams); - const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); - console.log(preAggregationsDescription); - expect(preAggregationsDescription.length).toBe(0); + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: ['cube_a.dim_a', 'cube_b.dim_b', 'cube_c.dim_c'], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); - expect(query.preAggregations?.preAggregationForQuery).toBeUndefined(); + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(preAggregationsDescription); + expect(preAggregationsDescription.length).toBe(0); - return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { - expect(res).toEqual( - [{ - cube_a__dim_a: 'dim_a', - cube_b__dim_b: 'dim_b', - cube_c__dim_c: 'dim_c', - }] - ); + expect(query.preAggregations?.preAggregationForQuery).toBeUndefined(); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual( + [{ + cube_a__dim_a: 'dim_a', + cube_b__dim_b: 'dim_b', + cube_c__dim_c: 'dim_c', + }] + ); + }); }); - }); + } }); From 3c391fd5091fde0a638488287bbe71ce0613736d Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 12:55:30 +0300 Subject: [PATCH 14/24] export type --- packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts index 39b9acc3296e7..505b10f952bf9 100644 --- a/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts +++ b/packages/cubejs-schema-compiler/src/compiler/JoinGraph.ts @@ -7,7 +7,7 @@ import type { CubeEvaluator, MeasureDefinition } from './CubeEvaluator'; import type { CubeDefinition, JoinDefinition } from './CubeSymbols'; import type { ErrorReporter } from './ErrorReporter'; -type JoinEdge = { +export type JoinEdge = { join: JoinDefinition, from: string, to: string, From de80e2f1108256df3a79234cd16dba69df606885 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 12:57:13 +0300 Subject: [PATCH 15/24] more types --- .../src/adapter/PreAggregations.ts | 22 ++++++++++++++----- 1 file changed, 17 insertions(+), 5 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 35332274af29f..a826eb4866ed2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1,6 +1,7 @@ import R from 'ramda'; import { CubeSymbols, PreAggregationDefinition } from '../compiler/CubeSymbols'; +import { FinishedJoinTree, JoinEdge } from '../compiler/JoinGraph'; import { UserError } from '../compiler/UserError'; import { BaseQuery } from './BaseQuery'; import { @@ -15,8 +16,6 @@ import { BaseGroupFilter } from './BaseGroupFilter'; import { BaseDimension } from './BaseDimension'; import { BaseSegment } from './BaseSegment'; -export type RollupJoin = any; - export type PartitionTimeDimension = { dimension: string; dateRange: [string, string]; @@ -45,6 +44,7 @@ export type PreAggregationForQuery = { references: PreAggregationReferences; preAggregationsToJoin?: PreAggregationForQuery[]; referencedPreAggregations?: PreAggregationForQuery[]; + // eslint-disable-next-line no-use-before-define rollupJoin?: RollupJoin; sqlAlias?: string; }; @@ -66,6 +66,18 @@ export type EvaluateReferencesContext = { export type BaseMember = BaseDimension | BaseMeasure | BaseFilter | BaseGroupFilter | BaseSegment; +export type JoinEdgeWithMembers = JoinEdge & { + fromMembers: string[]; + toMembers: string[]; +}; + +export type RollupJoinItem = JoinEdgeWithMembers & { + fromPreAggObj: PreAggregationForQuery; + toPreAggObj: PreAggregationForQuery; +}; + +export type RollupJoin = RollupJoinItem[]; + export type CanUsePreAggregationFn = (references: PreAggregationReferences) => boolean; /** @@ -1050,7 +1062,7 @@ export class PreAggregations { ); const existingJoins = R.unnest(preAggObjsToJoin.map( // TODO join hints? - p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))) + p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!) )); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && @@ -1088,7 +1100,7 @@ export class PreAggregations { return fromPreAggObj[0]; } - private resolveJoinMembers(join) { + private resolveJoinMembers(join: FinishedJoinTree): JoinEdgeWithMembers[] { return join.joins.map(j => { const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); @@ -1512,7 +1524,7 @@ export class PreAggregations { }); if (preAggregationForQuery.preAggregation.type === 'rollupJoin') { - const join = preAggregationForQuery.rollupJoin; + const join = preAggregationForQuery.rollupJoin!; toJoin = [ sqlAndAlias(join[0].fromPreAggObj), From 9f154d00f99c4b705689476ffc2e0f55c84fb913 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 13:38:10 +0300 Subject: [PATCH 16/24] build fullNames for rollupJoin/Lambda in the evaluatedPreAggregationObj() --- .../src/adapter/PreAggregations.ts | 182 ++++++++---------- 1 file changed, 83 insertions(+), 99 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index a826eb4866ed2..3467b4e9db5a2 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -92,6 +92,11 @@ export type FullPreAggregationDescription = any; */ export type TransformedQuery = any; +type BuildRollupJoinResult = { + rollupJoin: RollupJoin; + existingJoins: JoinEdgeWithMembers[]; +}; + export class PreAggregations { private readonly query: BaseQuery; @@ -521,18 +526,7 @@ export class PreAggregations { filterDimensionsSingleValueEqual = allValuesEq1(filterDimensionsSingleValueEqual) ? new Set(filterDimensionsSingleValueEqual?.keys()) : null; - // Build reverse query joins map, which is used for - // rollupLambda and rollupJoin pre-aggs matching later - const joinsMap: Record = {}; - if (query.join) { - for (const j of query.join.joins) { - joinsMap[j.to] = j.from; - } - } - return { - joinGraphRoot: query.join?.root, - joinsMap, sortedDimensions, sortedTimeDimensions, timeDimensions, @@ -651,16 +645,6 @@ export class PreAggregations { transformedQuery.allBackAliasMembers[r] || r )); - const buildPath = (cube: string): string[] => { - const path = [cube]; - const parentMap = transformedQuery.joinsMap; - while (parentMap[cube]) { - cube = parentMap[cube]; - path.push(cube); - } - return path.reverse(); - }; - /** * Determine whether pre-aggregation can be used or not. */ @@ -669,37 +653,9 @@ export class PreAggregations { ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - let dimsToMatch: string[]; - let measToMatch: string[]; - let timeDimsToMatch: PreAggregationTimeDimensionReference[]; - - if (references.rollups.length > 0) { - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - // but joinGraph of the query has all the connections, necessary for serving the query, - // so we use this information to complete the full paths of members from the root of the query - // up to the pre-agg cube. - // We use references from the underlying pre-aggregations, filtered with members existing in the root - // pre-aggregation itself. - - dimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameDimensions) - .filter(d => references.dimensions.some(rd => d.endsWith(rd))); - timeDimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameTimeDimensions) - .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); - measToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameMeasures) - .filter(m => references.measures.some(rm => m.endsWith(rm))); - } else { - dimsToMatch = references.fullNameDimensions; - timeDimsToMatch = references.fullNameTimeDimensions; - measToMatch = references.fullNameMeasures; - } - - const refTimeDimensions = backAlias(sortTimeDimensions(timeDimsToMatch)); - const backAliasMeasures = backAlias(measToMatch); - const backAliasDimensions = backAlias(dimsToMatch); + const refTimeDimensions = backAlias(sortTimeDimensions(references.fullNameTimeDimensions)); + const backAliasMeasures = backAlias(references.fullNameMeasures); + const backAliasDimensions = backAlias(references.fullNameDimensions); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -716,9 +672,9 @@ export class PreAggregations { transformedQuery.allFiltersWithinSelectedDimensions && R.equals(backAliasDimensions, transformedQuery.sortedDimensions) ) && ( - R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.measures) || + R.all(m => backAliasMeasures.includes(m), transformedQuery.measures) || // TODO do we need backAlias here? - R.all(m => backAliasMeasures.indexOf(m) !== -1, transformedQuery.leafMeasures) + R.all(m => backAliasMeasures.includes(m), transformedQuery.leafMeasures) )); }; @@ -790,36 +746,8 @@ export class PreAggregations { } } - let dimsToMatch: string[]; - let timeDimsToMatch: PreAggregationTimeDimensionReference[]; - - if (references.rollups.length > 0) { - // In 'rollupJoin' / 'rollupLambda' pre-aggregations fullName members will be empty, because there are - // no connections in the joinTree between cubes from different datasources - // but joinGraph of the query has all the connections, necessary for serving the query, - // so we use this information to complete the full paths of members from the root of the query - // up to the pre-agg cube. - // We use references from the underlying pre-aggregations, filtered with members existing in the root - // pre-aggregation itself. - - dimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameDimensions) - .filter(d => references.dimensions.some(rd => d.endsWith(rd))) - .map(d => { - const [cube, ...restPath] = d.split('.'); - if (cube === transformedQuery.joinGraphRoot) { - return d; - } - const path = buildPath(cube); - return `${path.join('.')}.${restPath.join('.')}`; - }); - timeDimsToMatch = references.rollupsReferences - .flatMap(rolRef => rolRef.fullNameTimeDimensions) - .filter(d => references.timeDimensions.some(rd => d.dimension.endsWith(rd.dimension))); - } else { - dimsToMatch = references.fullNameDimensions; - timeDimsToMatch = references.fullNameTimeDimensions; - } + const dimsToMatch = references.fullNameDimensions; + const timeDimsToMatch = references.fullNameTimeDimensions; const dimensionsMatch = (dimensions, doBackAlias) => { const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; @@ -1035,7 +963,7 @@ export class PreAggregations { } // TODO check multiplication factor didn't change - private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): RollupJoin { + private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): BuildRollupJoinResult { return this.query.cacheValue( ['buildRollupJoin', JSON.stringify(preAggObj), JSON.stringify(preAggObjsToJoin)], () => { @@ -1055,15 +983,22 @@ export class PreAggregations { return hints; }).flat(); - const targetJoins = this.resolveJoinMembers( - this.query.joinGraph.buildJoin( - preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) - ) + + const builtJoinTree = this.query.joinGraph.buildJoin( + preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) ); - const existingJoins = R.unnest(preAggObjsToJoin.map( - // TODO join hints? - p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!) - )); + + if (!builtJoinTree) { + throw new UserError(`Can't build join tree for pre-aggregation ${preAggObj.cube}.${preAggObj.preAggregationName}`); + } + + const targetJoins = this.resolveJoinMembers(builtJoinTree); + + // TODO join hints? + const existingJoins = preAggObjsToJoin + .map(p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!)) + .flat(); + const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( existing => existing.originalFrom === target.originalFrom && existing.originalTo === target.originalTo && @@ -1073,7 +1008,7 @@ export class PreAggregations { if (!nonExistingJoins.length) { throw new UserError(`Nothing to join in rollup join. Target joins ${JSON.stringify(targetJoins)} are included in existing rollup joins ${JSON.stringify(existingJoins)}`); } - return nonExistingJoins.map(join => { + const rollupJoin = nonExistingJoins.map(join => { const fromPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.fromMembers, join); const toPreAggObj = this.preAggObjForJoin(preAggObjsToJoin, join.toMembers, join); return { @@ -1082,6 +1017,11 @@ export class PreAggregations { toPreAggObj }; }); + + return { + rollupJoin, + existingJoins, + }; } ); } @@ -1142,8 +1082,8 @@ export class PreAggregations { preAggregationName, preAggregation, cube, - // For rollupJoin and rollupLambda we need to pass references of the underlying rollups - // to canUsePreAggregation fn, which are collected later; + // For rollupJoin and rollupLambda we need to enrich references with data + // from the underlying rollups which are collected later; canUsePreAggregation: preAggregation.type === 'rollup' ? canUsePreAggregation(references) : false, references, preAggregationId: `${cube}.${preAggregationName}` @@ -1165,12 +1105,53 @@ export class PreAggregations { preAggregationsToJoin.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); }); - const canUsePreAggregationResult = canUsePreAggregation(references); + const { rollupJoin, existingJoins } = this.buildRollupJoin(preAggObj, preAggregationsToJoin); + + const joinsMap: Record = {}; + for (const j of rollupJoin) { + joinsMap[j.to] = j.from; + } + for (const j of existingJoins) { + joinsMap[j.to] = j.from; + } + + const buildPath = (cubeName: string): string[] => { + const path = [cubeName]; + const parentMap = joinsMap; + while (parentMap[cubeName]) { + cubeName = parentMap[cubeName]; + path.push(cubeName); + } + return path.reverse(); + }; + + references.fullNameDimensions = references.dimensions.map(d => { + const [cubeName, ...restPath] = d.split('.'); + const path = buildPath(cubeName); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + references.fullNameMeasures = references.measures.map(m => { + const [cubeName, ...restPath] = m.split('.'); + const path = buildPath(cubeName); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + references.fullNameTimeDimensions = references.timeDimensions.map(td => { + const [cubeName, ...restPath] = td.dimension.split('.'); + const path = buildPath(cubeName); + + return { + ...td, + dimension: `${path.join('.')}.${restPath.join('.')}`, + }; + }); + return { ...preAggObj, - canUsePreAggregation: canUsePreAggregationResult, + canUsePreAggregation: canUsePreAggregation(references), preAggregationsToJoin, - rollupJoin: canUsePreAggregationResult ? this.buildRollupJoin(preAggObj, preAggregationsToJoin) : null, + rollupJoin, }; } else if (preAggregation.type === 'rollupLambda') { // TODO evaluation optimizations. Should be cached or moved to compile time. @@ -1217,6 +1198,9 @@ export class PreAggregations { }); referencedPreAggregations.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); + references.fullNameDimensions.push(...preAgg.references.fullNameDimensions); + references.fullNameMeasures.push(...preAgg.references.fullNameMeasures); + references.fullNameTimeDimensions.push(...preAgg.references.fullNameTimeDimensions); }); return { ...preAggObj, From fd912a1ebdd7966846c9d80b95f07d4cdec1c1bf Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 16:20:14 +0300 Subject: [PATCH 17/24] add memberShortNameFromPath() to Evaluator --- .../src/compiler/CubeEvaluator.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index f89ddad1da0db..6acda3f3e256b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -731,6 +731,18 @@ export class CubeEvaluator extends CubeSymbols { return !!this.evaluatedCubes[cube]; } + public memberShortNameFromPath(path: string | string[]): string { + if (!Array.isArray(path)) { + path = path.split('.'); + } + + if (path.length < 2) { + throw new UserError(`Not full member name provided: ${path[0]}`); + } + + return `${path.at(-2)}.${path.at(-1)}`; + } + public cubeFromPath(path: string): EvaluatedCube { return this.evaluatedCubes[this.cubeNameFromPath(path)]; } From b1f2f318c0a5eb9dc2b5ad4eff508d1d396eede9 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 16:20:37 +0300 Subject: [PATCH 18/24] fix type --- packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 3467b4e9db5a2..57ff7e5d4dbfc 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1654,8 +1654,7 @@ export class PreAggregations { private rollupMembers(preAggregationForQuery: PreAggregationForQuery, type: T): PreAggregationReferences[T] { return preAggregationForQuery.preAggregation.type === 'autoRollup' ? - // TODO proper types - (preAggregationForQuery.preAggregation as any)[type] : + preAggregationForQuery.preAggregation[type] : this.evaluateAllReferences(preAggregationForQuery.cube, preAggregationForQuery.preAggregation, preAggregationForQuery.preAggregationName)[type]; } From f5c5d4f5ea1d799d030c9df5d5f7bc22511366a3 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 17:02:30 +0300 Subject: [PATCH 19/24] get rid of references.fullName* in favor of fullpath-members --- .../src/adapter/PreAggregations.ts | 36 ++++++++----------- .../src/compiler/CubeEvaluator.ts | 6 ---- .../test/unit/pre-agg-by-filter-match.test.ts | 6 ---- .../test/unit/pre-agg-time-dim-match.test.ts | 6 ---- .../test/unit/RefreshScheduler.test.ts | 3 -- 5 files changed, 15 insertions(+), 42 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 57ff7e5d4dbfc..8a8bc44913bcd 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -653,9 +653,9 @@ export class PreAggregations { ? transformedQuery.timeDimensions : transformedQuery.sortedTimeDimensions; - const refTimeDimensions = backAlias(sortTimeDimensions(references.fullNameTimeDimensions)); - const backAliasMeasures = backAlias(references.fullNameMeasures); - const backAliasDimensions = backAlias(references.fullNameDimensions); + const refTimeDimensions = backAlias(sortTimeDimensions(references.timeDimensions)); + const backAliasMeasures = backAlias(references.measures); + const backAliasDimensions = backAlias(references.dimensions); return (( transformedQuery.hasNoTimeDimensionsWithoutGranularity ) && ( @@ -746,11 +746,8 @@ export class PreAggregations { } } - const dimsToMatch = references.fullNameDimensions; - const timeDimsToMatch = references.fullNameTimeDimensions; - const dimensionsMatch = (dimensions, doBackAlias) => { - const target = doBackAlias ? backAlias(dimsToMatch) : dimsToMatch; + const target = doBackAlias ? backAlias(references.dimensions) : references.dimensions; return dimensions.every(d => target.includes(d)); }; @@ -766,8 +763,8 @@ export class PreAggregations { ) )( doBackAlias ? - backAlias(sortTimeDimensions(timeDimsToMatch)) : - (sortTimeDimensions(timeDimsToMatch)) + backAlias(sortTimeDimensions(references.timeDimensions)) : + (sortTimeDimensions(references.timeDimensions)) ); if (transformedQuery.ungrouped) { @@ -1065,8 +1062,8 @@ export class PreAggregations { private cubesFromPreAggregation(preAggObj: PreAggregationForQuery): string[] { return R.uniq( - preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', m)).concat( - preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(m)) + preAggObj.references.measures.map(m => this.query.cubeEvaluator.parsePath('measures', this.query.cubeEvaluator.memberShortNameFromPath(m))).concat( + preAggObj.references.dimensions.map(m => this.query.cubeEvaluator.parsePathAnyType(this.query.cubeEvaluator.memberShortNameFromPath(m))) ).map(p => p[0]) ); } @@ -1125,19 +1122,19 @@ export class PreAggregations { return path.reverse(); }; - references.fullNameDimensions = references.dimensions.map(d => { + references.dimensions = references.dimensions.map(d => { const [cubeName, ...restPath] = d.split('.'); const path = buildPath(cubeName); return `${path.join('.')}.${restPath.join('.')}`; }); - references.fullNameMeasures = references.measures.map(m => { + references.measures = references.measures.map(m => { const [cubeName, ...restPath] = m.split('.'); const path = buildPath(cubeName); return `${path.join('.')}.${restPath.join('.')}`; }); - references.fullNameTimeDimensions = references.timeDimensions.map(td => { + references.timeDimensions = references.timeDimensions.map(td => { const [cubeName, ...restPath] = td.dimension.split('.'); const path = buildPath(cubeName); @@ -1198,9 +1195,6 @@ export class PreAggregations { }); referencedPreAggregations.forEach(preAgg => { references.rollupsReferences.push(preAgg.references); - references.fullNameDimensions.push(...preAgg.references.fullNameDimensions); - references.fullNameMeasures.push(...preAgg.references.fullNameMeasures); - references.fullNameTimeDimensions.push(...preAgg.references.fullNameTimeDimensions); }); return { ...preAggObj, @@ -1235,7 +1229,7 @@ export class PreAggregations { if (typeof member !== 'string') { return `${member.dimension.split('.')[1]}.${member.granularity}`; } else { - return member.split('.')[1]; + return member.split('.').at(-1)!; } }); } @@ -1386,9 +1380,9 @@ export class PreAggregations { // So we store full named members separately and use them in canUsePreAggregation functions. references.joinTree = preAggQuery.join; const root = references.joinTree?.root || ''; - references.fullNameMeasures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); - references.fullNameDimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); - references.fullNameTimeDimensions = references.timeDimensions.map(d => ({ + references.measures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); + references.dimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); + references.timeDimensions = references.timeDimensions.map(d => ({ dimension: (d.dimension.startsWith(root) ? d.dimension : `${root}.${d.dimension}`), granularity: d.granularity, })); diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 6acda3f3e256b..396640004034b 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -89,11 +89,8 @@ export type PreAggregationTimeDimensionReference = { export type PreAggregationReferences = { allowNonStrictDateRangeMatch?: boolean, dimensions: Array, - fullNameDimensions: Array, measures: Array, - fullNameMeasures: Array, timeDimensions: Array, - fullNameTimeDimensions: Array, rollups: Array, rollupsReferences: Array, multipliedMeasures?: Array, @@ -901,9 +898,6 @@ export class CubeEvaluator extends CubeSymbols { timeDimensions, rollups: aggregation.rollupReferences && this.evaluateReferences(cube, aggregation.rollupReferences, { originalSorting: true }) || [], - fullNameDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() - fullNameMeasures: [], // May be filled in PreAggregations.evaluateAllReferences() - fullNameTimeDimensions: [], // May be filled in PreAggregations.evaluateAllReferences() rollupsReferences: [], // May be filled in PreAggregations.evaluateAllReferences() }; } diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts index 7253ca3c35720..1b2d9f72af826 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-by-filter-match.test.ts @@ -59,12 +59,6 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: testPreAgg.segments ? testPreAgg.dimensions.concat(testPreAgg.segments) : testPreAgg.dimensions, - fullNameMeasures: testPreAgg.measures, - fullNameTimeDimensions: [{ - dimension: testPreAgg.timeDimension, - granularity: testPreAgg.granularity, - }], rollupsReferences: [], }; diff --git a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts index 56d294701ba6c..784f066ecd2ec 100644 --- a/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts +++ b/packages/cubejs-schema-compiler/test/unit/pre-agg-time-dim-match.test.ts @@ -69,12 +69,6 @@ describe('Pre Aggregation by filter match tests', () => { granularity: testPreAgg.granularity, }], rollups: [], - fullNameDimensions: testPreAgg.dimensions, - fullNameMeasures: testPreAgg.measures, - fullNameTimeDimensions: [{ - dimension: testPreAgg.timeDimension, - granularity: testPreAgg.granularity, - }], rollupsReferences: [], }; diff --git a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts index 370dd90d82742..22309182253cc 100644 --- a/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts +++ b/packages/cubejs-server-core/test/unit/RefreshScheduler.test.ts @@ -666,9 +666,6 @@ describe('Refresh Scheduler', () => { timeDimensions: [{ dimension: 'Foo.time', granularity: 'hour' }], rollups: [], rollupsReferences: [], - fullNameDimensions: [], - fullNameMeasures: [], - fullNameTimeDimensions: [], }, refreshKey: { every: '1 hour', updateWindow: '1 day', incremental: true }, }, From 6992e59da18b2b82862e40fe6569e18f7ec55658 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Wed, 22 Oct 2025 19:25:57 +0300 Subject: [PATCH 20/24] some refactoring to avoid copy/paste --- .../src/adapter/PreAggregations.ts | 128 +++++++++--------- .../src/compiler/CubeEvaluator.ts | 2 +- 2 files changed, 68 insertions(+), 62 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 8a8bc44913bcd..019223af7449f 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -172,8 +172,7 @@ export class PreAggregations { let preAggregations: PreAggregationForQuery[] = [foundPreAggregation]; if (foundPreAggregation.preAggregation.type === 'rollupJoin') { preAggregations = foundPreAggregation.preAggregationsToJoin || []; - } - if (foundPreAggregation.preAggregation.type === 'rollupLambda') { + } else if (foundPreAggregation.preAggregation.type === 'rollupLambda') { preAggregations = foundPreAggregation.referencedPreAggregations || []; } @@ -959,6 +958,20 @@ export class PreAggregations { } } + private collectJoinHintsFromRollupReferences(refs: PreAggregationReferences): (string | string[])[] { + if (!refs.joinTree) { + return []; + } + + const hints: (string | string[])[] = [refs.joinTree.root]; + + for (const j of refs.joinTree.joins) { + hints.push([j.from, j.to]); + } + + return hints; + } + // TODO check multiplication factor didn't change private buildRollupJoin(preAggObj: PreAggregationForQuery, preAggObjsToJoin: PreAggregationForQuery[]): BuildRollupJoinResult { return this.query.cacheValue( @@ -967,19 +980,9 @@ export class PreAggregations { // It's important to build join graph not only using the pre-agg members, but also // taking into account all explicit underlying rollup pre-aggregation joins, because // otherwise the built join tree might differ from the actual pre-aggregation. - const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map(r => { - if (!r.joinTree) { - return []; - } - - const hints: (string | string[])[] = [r.joinTree.root]; - - for (const j of r.joinTree.joins) { - hints.push([j.from, j.to]); - } - - return hints; - }).flat(); + const preAggJoinsJoinHints = preAggObj.references.rollupsReferences.map( + this.collectJoinHintsFromRollupReferences + ).flat(); const builtJoinTree = this.query.joinGraph.buildJoin( preAggJoinsJoinHints.concat(this.cubesFromPreAggregation(preAggObj)) @@ -1112,37 +1115,9 @@ export class PreAggregations { joinsMap[j.to] = j.from; } - const buildPath = (cubeName: string): string[] => { - const path = [cubeName]; - const parentMap = joinsMap; - while (parentMap[cubeName]) { - cubeName = parentMap[cubeName]; - path.push(cubeName); - } - return path.reverse(); - }; - - references.dimensions = references.dimensions.map(d => { - const [cubeName, ...restPath] = d.split('.'); - const path = buildPath(cubeName); - - return `${path.join('.')}.${restPath.join('.')}`; - }); - references.measures = references.measures.map(m => { - const [cubeName, ...restPath] = m.split('.'); - const path = buildPath(cubeName); - - return `${path.join('.')}.${restPath.join('.')}`; - }); - references.timeDimensions = references.timeDimensions.map(td => { - const [cubeName, ...restPath] = td.dimension.split('.'); - const path = buildPath(cubeName); - - return { - ...td, - dimension: `${path.join('.')}.${restPath.join('.')}`, - }; - }); + references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); + references.measures = this.buildMembersFullName(references.measures, joinsMap); + references.timeDimensions = this.buildTimeDimensionsFullName(references.timeDimensions, joinsMap); return { ...preAggObj, @@ -1315,11 +1290,10 @@ export class PreAggregations { cube, aggregation ) && - !!references.dimensions.find((d) => { + references.dimensions.some((d) => this.query.cubeEvaluator.dimensionByPath( // `d` can contain full join path, so we should trim it - const trimmedDimension = CubeSymbols.joinHintFromPath(d).path; - return this.query.cubeEvaluator.dimensionByPath(trimmedDimension).primaryKey; - }), + this.query.cubeEvaluator.memberShortNameFromPath(d) + ).primaryKey), }); } @@ -1364,6 +1338,38 @@ export class PreAggregations { .toLowerCase(); } + private enrichMembersCubeJoinPath(cubeName: string, joinsMap: Record): string[] { + const path = [cubeName]; + const parentMap = joinsMap; + while (parentMap[cubeName]) { + cubeName = parentMap[cubeName]; + path.push(cubeName); + } + + return path.reverse(); + } + + private buildMembersFullName(members: string[], joinsMap: Record): string[] { + return members.map(d => { + const [cubeName, ...restPath] = d.split('.'); + const path = this.enrichMembersCubeJoinPath(cubeName, joinsMap); + + return `${path.join('.')}.${restPath.join('.')}`; + }); + } + + private buildTimeDimensionsFullName(members: PreAggregationTimeDimensionReference[], joinsMap: Record): PreAggregationTimeDimensionReference[] { + return members.map(td => { + const [cubeName, ...restPath] = td.dimension.split('.'); + const path = this.enrichMembersCubeJoinPath(cubeName, joinsMap); + + return { + ...td, + dimension: `${path.join('.')}.${restPath.join('.')}`, + }; + }); + } + private evaluateAllReferences(cube: string, aggregation: PreAggregationDefinition, preAggregationName: string | null = null, context: EvaluateReferencesContext = {}): PreAggregationReferences { const evaluateReferences = () => { const references = this.query.cubeEvaluator.evaluatePreAggregationReferences(cube, aggregation); @@ -1374,18 +1380,18 @@ export class PreAggregations { if (preAggQuery) { // We need to build a join tree for all references, so they would always include full join path // even for preaggregation references without join path. It is necessary to be able to match - // query and preaggregation based on full join tree. But we can not update - // references.{dimensions,measures,timeDimensions} directly, because it will break - // evaluation of references in the query on later stages. - // So we store full named members separately and use them in canUsePreAggregation functions. + // query and preaggregation based on full join tree. references.joinTree = preAggQuery.join; - const root = references.joinTree?.root || ''; - references.measures = references.measures.map(m => (m.startsWith(root) ? m : `${root}.${m}`)); - references.dimensions = references.dimensions.map(d => (d.startsWith(root) ? d : `${root}.${d}`)); - references.timeDimensions = references.timeDimensions.map(d => ({ - dimension: (d.dimension.startsWith(root) ? d.dimension : `${root}.${d.dimension}`), - granularity: d.granularity, - })); + const joinsMap: Record = {}; + if (references.joinTree) { + for (const j of references.joinTree.joins) { + joinsMap[j.to] = j.from; + } + } + + references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap); + references.measures = this.buildMembersFullName(references.measures, joinsMap); + references.timeDimensions = this.buildTimeDimensionsFullName(references.timeDimensions, joinsMap); } } if (aggregation.type === 'rollupLambda') { diff --git a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts index 396640004034b..b7f85b84bf682 100644 --- a/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts +++ b/packages/cubejs-schema-compiler/src/compiler/CubeEvaluator.ts @@ -737,7 +737,7 @@ export class CubeEvaluator extends CubeSymbols { throw new UserError(`Not full member name provided: ${path[0]}`); } - return `${path.at(-2)}.${path.at(-1)}`; + return path.slice(-2).join('.'); } public cubeFromPath(path: string): EvaluatedCube { From de9c6b65794b492e0fc4580ddb580eddf9634194 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 17:09:34 +0300 Subject: [PATCH 21/24] add 'rollupJoin pre-aggregation matching with transitive joins' test --- .../postgres/pre-aggregations.test.ts | 261 ++++++++++++++++++ 1 file changed, 261 insertions(+) diff --git a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts index 66148cea4e02c..436f884a4b72d 100644 --- a/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts +++ b/packages/cubejs-schema-compiler/test/integration/postgres/pre-aggregations.test.ts @@ -925,6 +925,213 @@ describe('PreAggregations', () => { ] }); + // Models with transitive joins for rollupJoin matching + cube('merchant_dims', { + sql: \` + SELECT 101 AS merchant_sk, 'M1' AS merchant_id + UNION ALL + SELECT 102 AS merchant_sk, 'M2' AS merchant_id + \`, + + dimensions: { + merchant_sk: { + sql: 'merchant_sk', + type: 'number', + primary_key: true + }, + merchant_id: { + sql: 'merchant_id', + type: 'string' + } + } + }); + + cube('product_dims', { + sql: \` + SELECT 201 AS product_sk, 'P1' AS product_id + UNION ALL + SELECT 202 AS product_sk, 'P2' AS product_id + \`, + + dimensions: { + product_sk: { + sql: 'product_sk', + type: 'number', + primary_key: true + }, + product_id: { + sql: 'product_id', + type: 'string' + } + } + }); + + cube('merchant_and_product_dims', { + sql: \` + SELECT 'M1' AS merchant_id, 'P1' AS product_id, 'Organic' AS acquisition_channel, 'SOLD' AS status + UNION ALL + SELECT 'M1' AS merchant_id, 'P2' AS product_id, 'Paid' AS acquisition_channel, 'PAID' AS status + UNION ALL + SELECT 'M2' AS merchant_id, 'P1' AS product_id, 'Referral' AS acquisition_channel, 'RETURNED' AS status + \`, + + dimensions: { + product_id: { + sql: 'product_id', + type: 'string', + primary_key: true + }, + merchant_id: { + sql: 'merchant_id', + type: 'string', + primary_key: true + }, + status: { + sql: 'status', + type: 'string' + }, + acquisition_channel: { + sql: 'acquisition_channel', + type: 'string' + } + }, + + pre_aggregations: { + bridge_rollup: { + dimensions: [ + merchant_id, + product_id, + acquisition_channel, + status + ] + } + } + }); + + cube('other_facts', { + sql: \` + SELECT 1 AS id, 1 AS fact_id, 'OF1' AS fact + UNION ALL + SELECT 2 AS id, 2 AS fact_id, 'OF2' AS fact + UNION ALL + SELECT 3 AS id, 3 AS fact_id, 'OF3' AS fact + \`, + + dimensions: { + other_fact_id: { + sql: 'id', + type: 'number', + primary_key: true + }, + fact_id: { + sql: 'fact_id', + type: 'number' + }, + fact: { + sql: 'fact', + type: 'string' + } + }, + + pre_aggregations: { + bridge_rollup: { + dimensions: [ + fact_id, + fact + ] + } + } + + }); + + cube('test_facts', { + sql: \` + SELECT 1 AS id, 101 AS merchant_sk, 201 AS product_sk, 100 AS amount + UNION ALL + SELECT 2 AS id, 101 AS merchant_sk, 202 AS product_sk, 150 AS amount + UNION ALL + SELECT 3 AS id, 102 AS merchant_sk, 201 AS product_sk, 200 AS amount + \`, + + joins: { + merchant_dims: { + relationship: 'many_to_one', + sql: \`\${CUBE.merchant_sk} = \${merchant_dims.merchant_sk}\` + }, + product_dims: { + relationship: 'many_to_one', + sql: \`\${CUBE.product_sk} = \${product_dims.product_sk}\` + }, + // Transitive join - depends on merchant_dims and product_dims + merchant_and_product_dims: { + relationship: 'many_to_one', + sql: \`\${merchant_dims.merchant_id} = \${merchant_and_product_dims.merchant_id} AND \${product_dims.product_id} = \${merchant_and_product_dims.product_id}\` + }, + other_facts: { + relationship: 'one_to_many', + sql: \`\${CUBE.id} = \${other_facts.fact_id}\` + }, + }, + + dimensions: { + id: { + sql: 'id', + type: 'number', + primary_key: true + }, + merchant_sk: { + sql: 'merchant_sk', + type: 'number' + }, + product_sk: { + sql: 'product_sk', + type: 'number' + }, + acquisition_channel: { + sql: \`\${merchant_and_product_dims.acquisition_channel}\`, + type: 'string' + } + }, + + measures: { + amount_sum: { + sql: 'amount', + type: 'sum' + } + }, + + pre_aggregations: { + facts_rollup: { + dimensions: [ + id, + merchant_sk, + merchant_dims.merchant_sk, + merchant_dims.merchant_id, + merchant_and_product_dims.merchant_id, + product_sk, + product_dims.product_sk, + product_dims.product_id, + merchant_and_product_dims.product_id, + acquisition_channel, + merchant_and_product_dims.status + ] + }, + rollupJoinTransitive: { + type: 'rollupJoin', + dimensions: [ + merchant_sk, + product_sk, + merchant_and_product_dims.status, + other_facts.fact + ], + rollups: [ + facts_rollup, + other_facts.bridge_rollup + ] + } + } + }); + `); it('simple pre-aggregation', async () => { @@ -3234,4 +3441,58 @@ describe('PreAggregations', () => { }); }); } + + it('rollupJoin pre-aggregation matching with transitive joins', async () => { + await compiler.compile(); + + const query = new PostgresQuery({ joinGraph, cubeEvaluator, compiler }, { + dimensions: [ + 'test_facts.merchant_sk', + 'test_facts.product_sk', + 'merchant_and_product_dims.status', + 'other_facts.fact' + ], + timezone: 'America/Los_Angeles', + preAggregationsSchema: '' + }); + + const queryAndParams = query.buildSqlAndParams(); + console.log(queryAndParams); + const preAggregationsDescription: any = query.preAggregations?.preAggregationsDescription(); + console.log(JSON.stringify(preAggregationsDescription, null, 2)); + + // Verify that both rollups are included in the description + expect(preAggregationsDescription.length).toBe(2); + const factsRollup = preAggregationsDescription.find(p => p.preAggregationId === 'test_facts.facts_rollup'); + const bridgeRollup = preAggregationsDescription.find(p => p.preAggregationId === 'other_facts.bridge_rollup'); + expect(factsRollup).toBeDefined(); + expect(bridgeRollup).toBeDefined(); + + // Verify that the rollupJoin pre-aggregation can be used for the query + expect(query.preAggregations?.preAggregationForQuery?.canUsePreAggregation).toEqual(true); + expect(query.preAggregations?.preAggregationForQuery?.preAggregationName).toEqual('rollupJoinTransitive'); + + return dbRunner.evaluateQueryWithPreAggregations(query).then(res => { + expect(res).toEqual([ + { + merchant_and_product_dims__status: 'SOLD', + other_facts__fact: 'OF1', + test_facts__merchant_sk: 101, + test_facts__product_sk: 201, + }, + { + merchant_and_product_dims__status: 'PAID', + other_facts__fact: 'OF2', + test_facts__merchant_sk: 101, + test_facts__product_sk: 202, + }, + { + merchant_and_product_dims__status: 'RETURNED', + other_facts__fact: 'OF3', + test_facts__merchant_sk: 102, + test_facts__product_sk: 201, + }, + ]); + }); + }); }); From b10fced5bfeba32dd75663c94c65169b1fe9dd92 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 17:10:59 +0300 Subject: [PATCH 22/24] fix buildRollupJoin --- .../cubejs-schema-compiler/src/adapter/PreAggregations.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 019223af7449f..983f8e06523ec 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -996,7 +996,11 @@ export class PreAggregations { // TODO join hints? const existingJoins = preAggObjsToJoin - .map(p => this.resolveJoinMembers(this.query.joinGraph.buildJoin(this.cubesFromPreAggregation(p))!)) + .map(p => this.resolveJoinMembers( + this.query.joinGraph.buildJoin( + this.collectJoinHintsFromRollupReferences(p.references).concat(this.cubesFromPreAggregation(p)) + )! + )) .flat(); const nonExistingJoins = targetJoins.filter(target => !existingJoins.find( From 931fe528c0c9fa756bfdff46ef2c3c7d9e80c003 Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 17:13:13 +0300 Subject: [PATCH 23/24] fix resolveJoinMembers() --- .../src/adapter/PreAggregations.ts | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 983f8e06523ec..715792c803728 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -1045,13 +1045,18 @@ export class PreAggregations { } private resolveJoinMembers(join: FinishedJoinTree): JoinEdgeWithMembers[] { + const joinMap = new Set(); + return join.joins.map(j => { + joinMap.add(j.originalFrom); + const memberPaths = this.query.collectMemberNamesFor(() => this.query.evaluateSql(j.originalFrom, j.join.sql)).map(m => m.split('.')); - const invalidMembers = memberPaths.filter(m => m[0] !== j.originalFrom && m[0] !== j.originalTo); + + const invalidMembers = memberPaths.filter(m => !joinMap.has(m[0]) && m[0] !== j.originalTo); if (invalidMembers.length) { throw new UserError(`Members ${invalidMembers.join(', ')} in join from '${j.originalFrom}' to '${j.originalTo}' doesn't reference join cubes`); } - const fromMembers = memberPaths.filter(m => m[0] === j.originalFrom).map(m => m.join('.')); + const fromMembers = memberPaths.filter(m => joinMap.has(m[0])).map(m => m.join('.')); if (!fromMembers.length) { throw new UserError(`From members are not found in [${memberPaths.map(m => m.join('.')).join(', ')}] for join ${JSON.stringify(j)}. Please make sure join fields are referencing dimensions instead of columns.`); } @@ -1059,6 +1064,8 @@ export class PreAggregations { if (!toMembers.length) { throw new UserError(`To members are not found in [${memberPaths.map(m => m.join('.')).join(', ')}] for join ${JSON.stringify(j)}. Please make sure join fields are referencing dimensions instead of columns.`); } + joinMap.add(j.originalTo); + return { ...j, fromMembers, From 8ad21df374f7d320a8ab159ffb1cf718e098146e Mon Sep 17 00:00:00 2001 From: Konstantin Burkalev Date: Thu, 23 Oct 2025 18:40:26 +0300 Subject: [PATCH 24/24] implement sortMembersByJoinTree() --- .../src/adapter/PreAggregations.ts | 31 ++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts index 715792c803728..d9741756d76b0 100644 --- a/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts +++ b/packages/cubejs-schema-compiler/src/adapter/PreAggregations.ts @@ -669,7 +669,10 @@ export class PreAggregations { references.dimensions.length === filterDimensionsSingleValueEqual.size && R.all(d => filterDimensionsSingleValueEqual.has(d), backAliasDimensions) || transformedQuery.allFiltersWithinSelectedDimensions && - R.equals(backAliasDimensions, transformedQuery.sortedDimensions) + // references.dimensions might be reordered because of joinTree join order, + // so we need to compare without order here. + backAliasDimensions.length === transformedQuery.sortedDimensions.length && + R.equals(new Set(backAliasDimensions), new Set(transformedQuery.sortedDimensions)) ) && ( R.all(m => backAliasMeasures.includes(m), transformedQuery.measures) || // TODO do we need backAlias here? @@ -1398,6 +1401,32 @@ export class PreAggregations { for (const j of references.joinTree.joins) { joinsMap[j.to] = j.from; } + + // As full-path references may be passed to query options, + // it is important to sort them based on join tree order, + // because full-path names work as explicit join hints, + // and JoinGraph will take them as granted in the order of + // occurrence. But that might be incorrect for transitive-join cases. + const sortMembersByJoinTree = (members: string[]) => { + const joinOrder: Record = {}; + joinOrder[references.joinTree!.root] = 0; + for (const join of references.joinTree!.joins) { + const index = references.joinTree!.joins.indexOf(join); + joinOrder[join.to] = index + 1; + } + + members.sort((a, b) => { + const cubeA = a.split('.')[0]; + const cubeB = b.split('.')[0]; + const orderA = joinOrder[cubeA] ?? Infinity; + const orderB = joinOrder[cubeB] ?? Infinity; + + return orderA - orderB; + }); + }; + + sortMembersByJoinTree(references.dimensions); + sortMembersByJoinTree(references.measures); } references.dimensions = this.buildMembersFullName(references.dimensions, joinsMap);