From a6f548911a2c88bb1bd7dc9c062ea23760581126 Mon Sep 17 00:00:00 2001
From: Steve Bowser <stevewbowser@gmail.com>
Date: Fri, 1 Nov 2024 17:38:54 +0000
Subject: [PATCH] add option for including ticket comments from zendesk

---
 sources/zendesk/__init__.py | 18 ++++++++++++++++--
 sources/zendesk_pipeline.py | 16 ++++++++++++++++
 2 files changed, 32 insertions(+), 2 deletions(-)

diff --git a/sources/zendesk/__init__.py b/sources/zendesk/__init__.py
index d47fa1360..fb624db3b 100644
--- a/sources/zendesk/__init__.py
+++ b/sources/zendesk/__init__.py
@@ -222,6 +222,7 @@ def zendesk_support(
     pivot_ticket_fields: bool = True,
     start_date: Optional[TAnyDateTime] = DEFAULT_START_DATE,
     end_date: Optional[TAnyDateTime] = None,
+    include_ticket_comments: bool = False,
 ) -> Iterable[DltResource]:
     """
     Retrieves data from Zendesk Support for tickets, users, brands, organizations, and groups.
@@ -238,6 +239,9 @@ def zendesk_support(
         start_date: The start time of the range for which to load. Defaults to January 1st 2000.
         end_date: The end time of the range for which to load data.
             If end time is not provided, the incremental loading will be enabled and after initial run, only new data will be retrieved
+        include_ticket_comments: You can include comments in the ticket_events by setting this to True.
+            If set to False, any comment present in the ticket update is described only by boolean flags
+            'comment_present' and 'comment_public'. Defaults to False
 
     Returns:
         Sequence[DltResource]: Multiple dlt resources.
@@ -263,14 +267,21 @@ def ticket_events(
             end_value=end_date_ts,
             allow_external_schedulers=True,
         ),
+        include_ticket_comments: bool = False,
     ) -> Iterator[TDataItem]:
         # URL For ticket events
         # 'https://d3v-dlthub.zendesk.com/api/v2/incremental/ticket_events.json?start_time=946684800'
+
+        params = {"start_time": timestamp.last_value}
+        if include_ticket_comments:
+            # sideload the actual values of the comments
+            params["include"] = "comment_events"
+
         event_pages = zendesk_client.get_pages(
             "/api/v2/incremental/ticket_events.json",
             "ticket_events",
             PaginationType.STREAM,
-            params={"start_time": timestamp.last_value},
+            params=params,
         )
         for page in event_pages:
             yield page
@@ -414,7 +425,10 @@ def load_ticket_fields_state(
     # loading base tables
     resource_list = [
         ticket_fields_resource(zendesk_client=zendesk_client),
-        ticket_events(zendesk_client=zendesk_client),
+        ticket_events(
+            zendesk_client=zendesk_client,
+            include_ticket_comments=include_ticket_comments,
+        ),
         ticket_table(zendesk_client=zendesk_client, pivot_fields=pivot_ticket_fields),
         ticket_metric_table(zendesk_client=zendesk_client),
     ]
diff --git a/sources/zendesk_pipeline.py b/sources/zendesk_pipeline.py
index f6025dfe2..9789145bf 100644
--- a/sources/zendesk_pipeline.py
+++ b/sources/zendesk_pipeline.py
@@ -49,6 +49,22 @@ def load_support_with_pivoting() -> Any:
     return info
 
 
+def load_support_with_ticket_comments() -> Any:
+    """
+    Loads Zendesk Support data with ticket comments included. 
+    Simply done by setting the include_ticket_comments to true.
+    Loads only the base tables.
+    """
+    pipeline = dlt.pipeline(
+        pipeline_name="zendesk_support_pivoting",
+        destination="postgres",
+        dev_mode=False,
+    )
+    data = zendesk_support(load_all=False, include_ticket_comments=True)
+    info = pipeline.run(data=data)
+    return info
+
+
 def incremental_load_all_start_date() -> Any:
     """
     Implements incremental load when possible to Support, Chat and Talk Endpoints. The default behaviour gets data since the last load time saved in dlt state or