Use gzip compression

Author: PetrHeinz

pipeline.py

@@ -5,6 +5,7 @@
 from apache_beam.options.pipeline_options import PipelineOptions
 from apache_beam.transforms.window import FixedWindows
 import requests
+import gzip
 from typing import Dict, Any, List
 from datetime import timedelta
 
@@ -15,7 +16,8 @@ def __init__(self, source_token: str, ingesting_host: str, batch_size: int):
         self.batch_size = batch_size
         self.headers = {
             'Authorization': f'Bearer {source_token}',
-            'Content-Type': 'application/json'
+            'Content-Type': 'application/json',
+            'Content-Encoding': 'gzip'
         }
         self.batch = []
 
@@ -45,11 +47,15 @@ def finish_bundle(self):
 
     def _send_batch(self):
         try:
-            # Send batch to Better Stack
+            # Convert batch to JSON and compress with gzip
+            json_data = json.dumps(self.batch)
+            compressed_data = gzip.compress(json_data.encode('utf-8'))
+            
+            # Send compressed batch to Better Stack
             response = requests.post(
                 self.ingesting_url,
                 headers=self.headers,
-                json=self.batch
+                data=compressed_data
             )
 
             if response.status_code != 202: