Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 3 additions & 5 deletions posthog/warehouse/models/test/test_managed_viewset.py
Original file line number Diff line number Diff line change
Expand Up @@ -58,8 +58,6 @@ def test_sync_views_creates_views(self):
self.assertIsNotNone(view.columns)
self.assertIn("HogQLQuery", view.query.get("kind", "")) # type: ignore

# TODO: There's a bug here, these shouldn't include the weird `no_property` bit
# Will fix in a follow-up, not related to the changes that introduced this test
expected_view_names = sorted(
[
"revenue_analytics.all.revenue_analytics_charge",
Expand All @@ -70,13 +68,13 @@ def test_sync_views_creates_views(self):
"revenue_analytics.events.purchase.charge_events_revenue_view",
"revenue_analytics.events.purchase.customer_events_revenue_view",
"revenue_analytics.events.purchase.revenue_item_events_revenue_view",
"revenue_analytics.events.purchase_no_property.product_events_revenue_view",
"revenue_analytics.events.purchase_no_property.subscription_events_revenue_view",
"revenue_analytics.events.purchase.product_events_revenue_view",
"revenue_analytics.events.purchase.subscription_events_revenue_view",
"revenue_analytics.events.subscription_charge.charge_events_revenue_view",
"revenue_analytics.events.subscription_charge.customer_events_revenue_view",
"revenue_analytics.events.subscription_charge.product_events_revenue_view",
"revenue_analytics.events.subscription_charge.revenue_item_events_revenue_view",
"revenue_analytics.events.subscription_charge_no_property.subscription_events_revenue_view",
"revenue_analytics.events.subscription_charge.subscription_events_revenue_view",
]
)

Expand Down
10 changes: 2 additions & 8 deletions products/revenue_analytics/backend/views/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ class BuiltQuery:
prefix: str
# HogQL AST for the view
query: ast.Expr
# Useful for debugging purposes, only asserted by in tests
test_comments: str | None = None


# A builder is a function that takes a SourceHandle and returns a single BuiltQuery object
Expand All @@ -44,11 +46,3 @@ def view_prefix_for_source(source: ExternalDataSource) -> str:
return source.source_type.lower()
prefix = source.prefix.strip("_")
return f"{source.source_type.lower()}.{prefix}"


def view_name_for_event(event: str, suffix: str) -> str:
return f"{view_prefix_for_event(event)}.{suffix}"


def view_name_for_source(source: ExternalDataSource, suffix: str) -> str:
return f"{view_prefix_for_source(source)}.{suffix}"
13 changes: 4 additions & 9 deletions products/revenue_analytics/backend/views/orchestrator.py
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,7 @@

from collections import defaultdict
from collections.abc import Iterable
from typing import Optional, cast
from typing import Optional

from django.db.models import Prefetch

Expand All @@ -18,12 +18,7 @@
from posthog.warehouse.types import ExternalDataSourceType

from products.revenue_analytics.backend.views import KIND_TO_CLASS, RevenueAnalyticsBaseView
from products.revenue_analytics.backend.views.core import (
BuiltQuery,
SourceHandle,
view_name_for_event,
view_name_for_source,
)
from products.revenue_analytics.backend.views.core import BuiltQuery, SourceHandle
from products.revenue_analytics.backend.views.schemas import SCHEMAS
from products.revenue_analytics.backend.views.sources.registry import BUILDERS

Expand Down Expand Up @@ -60,9 +55,9 @@ def _query_to_view(

if handle.source is not None:
id = query.key # Stable key (i.e. table.id)
name = view_name_for_source(cast(ExternalDataSource, handle.source), schema.source_suffix)
name = f"{query.prefix}.{schema.source_suffix}"
else:
id = name = view_name_for_event(query.key, schema.events_suffix)
id = name = f"{query.prefix}.{schema.events_suffix}"

return view_cls(
id=id,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,10 @@ def build(handle: SourceHandle) -> BuiltQuery:

if not event.productProperty:
return BuiltQuery(
key=f"{event.eventName}.no_property",
key=event.eventName,
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(PRODUCT_SCHEMA.fields.keys())),
test_comments="no_property",
)

events_query = ast.SelectQuery(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -18,9 +18,10 @@ def build(handle: SourceHandle) -> BuiltQuery:

if event.subscriptionProperty is None:
return BuiltQuery(
key=f"{event.eventName}.no_property",
key=event.eventName,
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SUBSCRIPTION_SCHEMA.fields.keys())),
test_comments="no_property",
)

events_query = ast.SelectQuery(
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,13 +29,19 @@ def build(handle: SourceHandle) -> BuiltQuery:
charge_schema = next((schema for schema in schemas if schema.name == STRIPE_CHARGE_RESOURCE_NAME), None)
if charge_schema is None:
return BuiltQuery(
key=f"{prefix}.no_source", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found yet
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_schema",
)

charge_schema = cast(ExternalDataSchema, charge_schema)
if charge_schema.table is None:
return BuiltQuery(
key=f"{prefix}.no_table", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_table",
)

table = cast(DataWarehouseTable, charge_schema.table)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -27,13 +27,19 @@ def build(handle: SourceHandle) -> BuiltQuery:
customer_schema = next((schema for schema in schemas if schema.name == STRIPE_CUSTOMER_RESOURCE_NAME), None)
if customer_schema is None:
return BuiltQuery(
key=f"{prefix}.no_source", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found yet
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_schema",
)

customer_schema = cast(ExternalDataSchema, customer_schema)
if customer_schema.table is None:
return BuiltQuery(
key=f"{prefix}.no_table", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_table",
)

invoice_schema = next((schema for schema in schemas if schema.name == STRIPE_INVOICE_RESOURCE_NAME), None)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,13 +23,19 @@ def build(handle: SourceHandle) -> BuiltQuery:
product_schema = next((schema for schema in schemas if schema.name == STRIPE_PRODUCT_RESOURCE_NAME), None)
if product_schema is None:
return BuiltQuery(
key=f"{prefix}.no_source", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found yet
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_schema",
)

product_schema = cast(ExternalDataSchema, product_schema)
if product_schema.table is None:
return BuiltQuery(
key=f"{prefix}.no_table", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_table",
)

table = cast(DataWarehouseTable, product_schema.table)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,7 +113,10 @@ def build(handle: SourceHandle) -> BuiltQuery:

if invoice_schema is None and charge_schema is None:
return BuiltQuery(
key=f"{prefix}.no_source", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found yet
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_schema",
)

invoice_table: DataWarehouseTable | None = None
Expand All @@ -131,7 +134,10 @@ def build(handle: SourceHandle) -> BuiltQuery:
team = charge_table.team
else:
return BuiltQuery(
key=f"{prefix}.no_table", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_table",
)

# Build the query for invoice items with revenue recognition splitting
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,13 +26,19 @@ def build(handle: SourceHandle) -> BuiltQuery:
subscription_schema = next((schema for schema in schemas if schema.name == STRIPE_SUBSCRIPTION_RESOURCE_NAME), None)
if subscription_schema is None:
return BuiltQuery(
key=f"{prefix}.no_source", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found yet
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_schema",
)

subscription_schema = cast(ExternalDataSchema, subscription_schema)
if subscription_schema.table is None:
return BuiltQuery(
key=f"{prefix}.no_table", prefix=prefix, query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys()))
key=str(source.id), # Using source rather than table because table hasn't been found
prefix=prefix,
query=ast.SelectQuery.empty(columns=list(SCHEMA.fields.keys())),
test_comments="no_table",
)

table = cast(DataWarehouseTable, subscription_schema.table)
Expand Down
12 changes: 11 additions & 1 deletion products/revenue_analytics/backend/views/sources/test/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,14 @@ def setUp(self):
super().setUp()
# Common setup for revenue analytics tests can go here

def assertBuiltQueryStructure(self, built_query: BuiltQuery, expected_key: str, expected_prefix: str):
def assertBuiltQueryStructure(
self,
built_query: BuiltQuery,
expected_key: str,
expected_prefix: str,
*,
expected_test_comments: str | None = "<SENTINEL_VALUE>",
):
"""
Assert that a BuiltQuery has the expected structure.

Expand All @@ -41,6 +48,9 @@ def assertBuiltQueryStructure(self, built_query: BuiltQuery, expected_key: str,
self.assertEqual(built_query.key, expected_key)
self.assertEqual(built_query.prefix, expected_prefix)

if expected_test_comments != "<SENTINEL_VALUE>":
self.assertEqual(built_query.test_comments, expected_test_comments)

def assertQueryContainsFields(self, query: ast.Expr, schema: Schema):
"""
Assert that a SelectQuery contains all expected fields in its select clause and that they appear in the same order.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,22 @@ def test_build_product_queries_even_for_events_without_product_property(self):
# Test purchase event (no productProperty)
handle = SourceHandle(type="events", team=self.team, event=self.events[0])
query = build(handle)
key = f"{self.PURCHASE_EVENT_NAME}.no_property"
self.assertBuiltQueryStructure(query, key, "revenue_analytics.events.purchase")
self.assertBuiltQueryStructure(
query,
self.PURCHASE_EVENT_NAME,
"revenue_analytics.events.purchase",
expected_test_comments="no_property",
)

query_sql = query.query.to_hogql()
self.assertQueryMatchesSnapshot(query_sql, replace_all_numbers=True)

# Test subscription_charge event (has productProperty)
handle = SourceHandle(type="events", team=self.team, event=self.events[1])
query = build(handle)
key = self.SUBSCRIPTION_CHARGE_EVENT_NAME
self.assertBuiltQueryStructure(
query,
key,
self.SUBSCRIPTION_CHARGE_EVENT_NAME,
"revenue_analytics.events.subscription_charge",
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,22 @@ def test_build_subscription_queries_even_for_events_without_subscription_propert
# Test purchase event (no subscriptionProperty)
handle = SourceHandle(type="events", team=self.team, event=self.events[0])
query = build(handle)
key = f"{self.PURCHASE_EVENT_NAME}.no_property"
self.assertBuiltQueryStructure(query, key, "revenue_analytics.events.purchase")
self.assertBuiltQueryStructure(
query,
self.PURCHASE_EVENT_NAME,
"revenue_analytics.events.purchase",
expected_test_comments="no_property",
)

query_sql = query.query.to_hogql()
self.assertQueryMatchesSnapshot(query_sql, replace_all_numbers=True)

# Test subscription_charge event (has subscriptionProperty)
handle = SourceHandle(type="events", team=self.team, event=self.events[1])
key = self.SUBSCRIPTION_CHARGE_EVENT_NAME
query = build(handle)
self.assertBuiltQueryStructure(
cast(BuiltQuery, query),
key,
self.SUBSCRIPTION_CHARGE_EVENT_NAME,
"revenue_analytics.events.subscription_charge",
)

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ def test_build_with_no_charge_schema(self):
self.assertQueryContainsFields(query.query, CHARGE_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_source",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_schema",
)

# Print and snapshot the generated HogQL query
Expand All @@ -62,8 +63,9 @@ def test_build_with_charge_schema_but_no_table(self):
self.assertQueryContainsFields(query.query, CHARGE_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_table",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_table",
)

# Print and snapshot the generated HogQL query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,8 +52,9 @@ def test_build_with_no_customer_schema(self):
self.assertQueryContainsFields(query.query, CUSTOMER_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_source",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_schema",
)

# Print and snapshot the generated HogQL query
Expand All @@ -76,8 +77,9 @@ def test_build_with_customer_schema_but_no_table(self):
self.assertQueryContainsFields(query.query, CUSTOMER_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_table",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_table",
)

# Print and snapshot the generated HogQL query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,9 @@ def test_build_with_no_product_schema(self):
self.assertQueryContainsFields(query.query, PRODUCT_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_source",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_schema",
)

# Print and snapshot the generated HogQL query
Expand All @@ -62,8 +63,9 @@ def test_build_with_product_schema_but_no_table(self):
self.assertQueryContainsFields(query.query, PRODUCT_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_table",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_table",
)

# Print and snapshot the generated HogQL query
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,9 @@ def test_build_with_no_relevant_schemas(self):
self.assertQueryContainsFields(query.query, REVENUE_ITEM_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_source",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_schema",
)
# Print and snapshot the generated HogQL query
self.assertQueryMatchesSnapshot(query.query.to_hogql(), replace_all_numbers=True)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -35,8 +35,9 @@ def test_build_with_no_subscription_schema(self):
self.assertQueryContainsFields(query.query, SUBSCRIPTION_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_source",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_schema",
)

# Print and snapshot the generated HogQL query
Expand All @@ -59,8 +60,9 @@ def test_build_with_subscription_schema_but_no_table(self):
self.assertQueryContainsFields(query.query, SUBSCRIPTION_SCHEMA)
self.assertBuiltQueryStructure(
query,
f"stripe.{self.external_data_source.prefix}.no_table",
str(self.stripe_handle.source.id), # type: ignore
f"stripe.{self.external_data_source.prefix}",
expected_test_comments="no_table",
)

# Print and snapshot the generated HogQL query
Expand Down
Loading
Loading