generator: allow outputing main_host once per interval or weekly (#204)

* generator: only output main_host once per interval

prevents having to parse 31M host entries when generating 1M

* generator: allow setting main_host output frequency

MAIN_HOST_FREQUENCY=1 - default, output every day
MAIN_HOST_FREQUENCY=7 - output every 7 days
MAIN_HOST_FREQUENCY=0 - output once, at the end of the interval

and update run-perf-gen to use 1, 7, 0 for 10k, 100k and 1M respectively

Author: himdel

tools/perf/generator.py

@@ -252,6 +252,7 @@ def parse_env(self):
         self.main_host = (
             int(os.getenv('MAIN_HOST_SIZE', '10000')),
             int(os.getenv('MAIN_HOST_UNIQUE_SIZE', '2000')),
+            int(os.getenv('MAIN_HOST_FREQUENCY', '1')),  # every N days; or once when 0
         )
         self.main_indirectmanagednodeaudit = (
             int(os.getenv('MAIN_INDIRECT_SIZE', '10000')),
@@ -287,6 +288,7 @@ def parse_args(self):
     MAIN_JOBHOSTSUMMARY_UNIQUE_SIZE (default: 2000)
     MAIN_HOST_SIZE (default: 10000)
     MAIN_HOST_UNIQUE_SIZE (default: 2000)
+    MAIN_HOST_FREQUENCY (default: 1)
     MAIN_INDIRECT_SIZE (default: 10000)
     MAIN_INDIRECT_UNIQUE_SIZE (default: 2000)
     MAIN_JOBEVENT_SIZE (default: 10000)
@@ -400,11 +402,25 @@ def save_tarballs(self, table):
             return
 
         if table == 'main_host':
-            # main_host - only generate csvs once, not filtered by since/until; use for each daily tarball
+            # main_host - only generate csvs once, not filtered by since/until
             with tempfile.TemporaryDirectory(prefix=f'metrics-generator-save-{table}') as temp_dir:
                 file_list = self.save_csvs(table, temp_dir, df)
 
+                # output every N days (MAIN_HOST_FREQUENCY=1), or at the end of the period if 0
+                frequency = self.main_host[2]
+                idx = 0
                 for since, until in daily_slicing(since=self.output_from, until=self.output_to):
+                    idx += 1
+                    if not frequency:
+                        continue
+                    if idx % frequency:
+                        continue
+
+                    logger.info(f'{table} - {since}-{until}')
+                    for file in file_list:
+                        self.tarify(table, since, until, file)
+
+                if not frequency:
                     logger.info(f'{table} - {since}-{until}')
                     for file in file_list:
                         self.tarify(table, since, until, file)

tools/perf/run-perf-gen

@@ -19,6 +19,7 @@ run_generator() {
   export MAIN_JOBHOSTSUMMARY_UNIQUE_SIZE="$2"
   export MAIN_HOST_SIZE="$1"
   export MAIN_HOST_UNIQUE_SIZE="$2"
+  export MAIN_HOST_FREQUENCY="$5"
   export MAIN_INDIRECT_SIZE="$1"
   export MAIN_INDIRECT_UNIQUE_SIZE="$2"
   export MAIN_JOBEVENT_SIZE="$1"
@@ -35,13 +36,13 @@ run_generator() {
 
 ## generator
 log "gen 10,000"
-run_generator 10000 2000 2026-01-01 2026-02-01
+run_generator 10000 2000 2026-01-01 2026-02-01 1
 
 log "gen 100,000"
-run_generator 100000 20000 2026-02-01 2026-03-01
+run_generator 100000 20000 2026-02-01 2026-03-01 7
 
 log "gen 1,000,000"
-run_generator 1000000 200000 2026-03-01 2026-04-01
+run_generator 1000000 200000 2026-03-01 2026-04-01 0
 
 ## checkup
 set -x