add support for including private Slack channels in slack source

mypy.ini

@@ -1,5 +1,5 @@
 [mypy]
-python_version=3.8
+python_version=3.9
 ignore_missing_imports=false
 strict_optional=false
 warn_redundant_casts=true

sources/slack/__init__.py

@@ -26,6 +26,7 @@ def slack_source(
     selected_channels: Optional[List[str]] = dlt.config.value,
     table_per_channel: bool = True,
     replies: bool = False,
+    include_private_channels: bool = False,
 ) -> Iterable[DltResource]:
     """
     The source for the Slack pipeline. Available resources are conversations, conversations_history
@@ -40,6 +41,8 @@ def slack_source(
         table_per_channel: Boolean flag, True by default. If True - for each channel separate table with messages is created.
             Otherwise, all messages are put in one table.
         replies: Boolean flag indicating if you want a replies table to be present as well. False by default.
+        include_private_channels: Boolean flag indicating if you want to include private channels and group DMs.
+            Defaults to False. Requires appropriate OAuth scopes (groups:read, mpim:read).
 
     Returns:
         Iterable[DltResource]: A list of DltResource objects representing the data resources.
@@ -57,23 +60,34 @@ def slack_source(
     )
 
     def get_channels(
-        slack_api: SlackAPI, selected_channels: Optional[List[str]]
+        slack_api: SlackAPI,
+        selected_channels: Optional[List[str]],
+        include_private_channels: bool = False,
     ) -> Tuple[List[TDataItem], List[TDataItem]]:
         """
         Returns channel fetched from slack and list of selected channels.
 
         Args:
             slack_api: Slack API instance.
             selected_channels: List of selected channels names or None.
+            include_private_channels: Whether to include private channels and group DMs.
 
         Returns:
             Tuple[List[TDataItem], List[TDataItem]]: fetched channels and selected fetched channels.
         """
         channels: List[TDataItem] = []
+
+        # Define conversation types based on the include_private_channels parameter
+        if include_private_channels:
+            conversation_types = "public_channel,private_channel,mpim"
+        else:
+            conversation_types = "public_channel"
+
         for page_data in slack_api.get_pages(
             resource="conversations.list",
             response_path="$.channels[*]",
             datetime_fields=DEFAULT_DATETIME_FIELDS,
+            params={"types": conversation_types},
         ):
             channels.extend(page_data)
 
@@ -87,7 +101,9 @@ def get_channels(
             fetch_channels = channels
         return channels, fetch_channels
 
-    channels, fetched_selected_channels = get_channels(api, selected_channels)
+    channels, fetched_selected_channels = get_channels(
+        api, selected_channels, include_private_channels
+    )
 
     @dlt.resource(name="channels", primary_key="id", write_disposition="replace")
     def channels_resource() -> Iterable[TDataItem]: