From 5a1431e6f999dee46e4742459ef563b55b4d229e Mon Sep 17 00:00:00 2001 From: filipecosta90 Date: Wed, 2 Apr 2025 10:58:03 +0100 Subject: [PATCH] Ensuring architecture is present on metadata across all timeseries --- pyproject.toml | 2 +- redisbench_admin/compare/args.py | 12 +++++ redisbench_admin/compare/compare.py | 57 ++++++++++++++++++++--- redisbench_admin/run/metrics.py | 51 +++++++++++--------- redisbench_admin/run/run.py | 2 +- redisbench_admin/run_remote/run_remote.py | 29 ++++++++++-- 6 files changed, 117 insertions(+), 36 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index b765830..19eb630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "redisbench-admin" -version = "0.11.32" +version = "0.11.35" description = "Redis benchmark run helper. A wrapper around Redis and Redis Modules benchmark tools ( ftsb_redisearch, memtier_benchmark, redis-benchmark, aibench, etc... )." authors = ["filipecosta90 ","Redis Performance Group "] readme = "README.md" diff --git a/redisbench_admin/compare/args.py b/redisbench_admin/compare/args.py index 8378fce..5aba32c 100644 --- a/redisbench_admin/compare/args.py +++ b/redisbench_admin/compare/args.py @@ -92,6 +92,18 @@ def create_compare_arguments(parser): default=1, help="Use the last N samples for each time-serie. by default will use last value only", ) + parser.add_argument( + "--first_n_baseline", + type=int, + default=-1, + help="Use the last N samples for each time-serie. by default will use last 7 available values", + ) + parser.add_argument( + "--first_n_comparison", + type=int, + default=-1, + help="Use the last N samples for each time-serie. by default will use last value only", + ) parser.add_argument( "--from-date", type=lambda s: datetime.datetime.strptime(s, "%Y-%m-%d"), diff --git a/redisbench_admin/compare/compare.py b/redisbench_admin/compare/compare.py index 6d79072..e489f9a 100644 --- a/redisbench_admin/compare/compare.py +++ b/redisbench_admin/compare/compare.py @@ -135,10 +135,31 @@ def compare_command_logic(args, project_name, project_version): last_n_baseline = args.last_n_baseline if last_n_comparison < 0: last_n_comparison = args.last_n_comparison - logging.info("Using last {} samples for baseline analysis".format(last_n_baseline)) - logging.info( - "Using last {} samples for comparison analysis".format(last_n_comparison) - ) + first_n_baseline = args.first_n_baseline + first_n_comparison = args.first_n_comparison + # Log the interval of values considered + if first_n_baseline >= 0: + logging.info( + "Using samples in the range [{}:{}] for baseline analysis".format( + first_n_baseline, last_n_baseline + ) + ) + else: + logging.info( + "Using last {} samples for baseline analysis".format(last_n_baseline) + ) + + if first_n_comparison >= 0: + logging.info( + "Using samples in the range [{}:{}] for comparison analysis".format( + first_n_comparison, last_n_comparison + ) + ) + else: + logging.info( + "Using last {} samples for comparison analysis".format(last_n_comparison) + ) + verbose = args.verbose regressions_percent_lower_limit = args.regressions_percent_lower_limit metric_name = args.metric_name @@ -280,6 +301,8 @@ def compare_command_logic(args, project_name, project_version): running_platform, baseline_architecture, comparison_architecture, + first_n_baseline, + first_n_comparison, ) comment_body = "" if total_comparison_points > 0: @@ -506,6 +529,8 @@ def compute_regression_table( running_platform=None, baseline_architecture=ARCH_X86, comparison_architecture=ARCH_X86, + first_n_baseline=-1, + first_n_comparison=-1, ): START_TIME_NOW_UTC, _, _ = get_start_time_vars() START_TIME_LAST_MONTH_UTC = START_TIME_NOW_UTC - datetime.timedelta(days=31) @@ -594,6 +619,8 @@ def compute_regression_table( running_platform, baseline_architecture, comparison_architecture, + first_n_baseline, + first_n_comparison, ) logging.info( "Printing differential analysis between {} and {}".format( @@ -723,6 +750,8 @@ def from_rts_to_regression_table( running_platform=None, baseline_architecture=ARCH_X86, comparison_architecture=ARCH_X86, + first_n_baseline=-1, + first_n_comparison=-1, ): print_all = print_regressions_only is False and print_improvements_only is False table = [] @@ -835,6 +864,7 @@ def from_rts_to_regression_table( largest_variance, last_n_baseline, verbose, + first_n_baseline, ) for ts_name_comparison in comparison_timeseries: datapoints_inner = rts.ts().revrange( @@ -854,6 +884,7 @@ def from_rts_to_regression_table( largest_variance, last_n_comparison, verbose, + first_n_comparison, ) waterline = regressions_percent_lower_limit @@ -1051,13 +1082,25 @@ def get_v_pct_change_and_largest_var( largest_variance, last_n=-1, verbose=False, + first_n=-1, ): comparison_nsamples = len(comparison_datapoints) if comparison_nsamples > 0: _, comparison_v = comparison_datapoints[0] - for tuple in comparison_datapoints: - if last_n < 0 or (last_n > 0 and len(comparison_values) < last_n): - comparison_values.append(tuple[1]) + + # Apply first_n and last_n boundaries + start_idx = 0 if first_n < 0 else max(0, min(first_n, comparison_nsamples)) + end_idx = ( + comparison_nsamples + if last_n < 0 + else max(0, min(last_n, comparison_nsamples)) + ) + + selected_data = comparison_datapoints[start_idx:end_idx] + + for tuple in selected_data: + comparison_values.append(tuple[1]) + comparison_df = pd.DataFrame(comparison_values) comparison_median = float(comparison_df.median()) comparison_v = comparison_median diff --git a/redisbench_admin/run/metrics.py b/redisbench_admin/run/metrics.py index 1a15270..3db76cc 100644 --- a/redisbench_admin/run/metrics.py +++ b/redisbench_admin/run/metrics.py @@ -5,7 +5,7 @@ # import logging import datetime as dt - +import redis from jsonpath_ng import parse @@ -99,28 +99,33 @@ def collect_redis_metrics( for conn_n, conn in enumerate(redis_conns): conn_res = {} for section in sections: - info = conn.info(section) - conn_res[section] = info - if section not in overall: - overall[section] = {} - for k, v in info.items(): - collect = True - if section_filter is not None: - if section in section_filter: - if k not in section_filter[section]: - collect = False - if collect and type(v) is float or type(v) is int: - if k not in overall[section]: - overall[section][k] = 0 - overall[section][k] += v - if collect and type(v) is dict: - for inner_k, inner_v in v.items(): - if type(inner_v) is float or type(inner_v) is int: - final_str_k = "{}_{}".format(k, inner_k) - if multi_shard: - final_str_k += "_shard_{}".format(conn_n + 1) - if final_str_k not in overall[section]: - overall[section][final_str_k] = inner_v + try: + info = conn.info(section) + conn_res[section] = info + if section not in overall: + overall[section] = {} + for k, v in info.items(): + collect = True + if section_filter is not None: + if section in section_filter: + if k not in section_filter[section]: + collect = False + if collect and type(v) is float or type(v) is int: + if k not in overall[section]: + overall[section][k] = 0 + overall[section][k] += v + if collect and type(v) is dict: + for inner_k, inner_v in v.items(): + if type(inner_v) is float or type(inner_v) is int: + final_str_k = "{}_{}".format(k, inner_k) + if multi_shard: + final_str_k += "_shard_{}".format(conn_n + 1) + if final_str_k not in overall[section]: + overall[section][final_str_k] = inner_v + except redis.exceptions.ConnectionError as e: + logging.warning( + f"Unable to collect section {section} from redis. error: {e.__str__}" + ) res.append(conn_res) diff --git a/redisbench_admin/run/run.py b/redisbench_admin/run/run.py index 5be95df..30659b0 100644 --- a/redisbench_admin/run/run.py +++ b/redisbench_admin/run/run.py @@ -86,7 +86,7 @@ def define_benchmark_plan(benchmark_definitions, default_specs): setup_contains_dbconfig = False if "dbconfig" in setup_settings: setup_contains_dbconfig = True - logging.info( + logging.debug( f"setup ({setup_name}): {setup_settings}. contains dbconfig {setup_contains_dbconfig}" ) diff --git a/redisbench_admin/run_remote/run_remote.py b/redisbench_admin/run_remote/run_remote.py index a2bfd15..ed1a1a2 100644 --- a/redisbench_admin/run_remote/run_remote.py +++ b/redisbench_admin/run_remote/run_remote.py @@ -134,7 +134,7 @@ def run_remote_command_logic(args, project_name, project_version): keep_env_and_topo = args.keep_env_and_topo skip_remote_db_setup = args.skip_db_setup flushall_on_every_test_start = args.flushall_on_every_test_start - redis_7 = args.redis_7 + redis_7 = True cluster_start_port = 20000 redis_password = args.db_pass ignore_keyspace_errors = args.ignore_keyspace_errors @@ -317,6 +317,7 @@ def run_remote_command_logic(args, project_name, project_version): spot_instance_error = False ts_key_spot_price = f"ts:{tf_triggering_env}:tests:spot_price" ts_key_full_price = f"ts:{tf_triggering_env}:tests:full_price" + ts_key_architecture = f"ts:{tf_triggering_env}:tests:arch:{architecture}" for benchmark_type, bench_by_dataset_map in benchmark_runs_plan.items(): if return_code != 0 and args.fail_fast: @@ -365,6 +366,8 @@ def run_remote_command_logic(args, project_name, project_version): ) continue metadata_tags = get_metadata_tags(benchmark_config) + if "arch" not in metadata_tags: + metadata_tags["arch"] = architecture logging.info( "Including the extra metadata tags into this test generated time-series: {}".format( metadata_tags @@ -462,6 +465,15 @@ def run_remote_command_logic(args, project_name, project_version): testcase_start_time_str, ) = get_start_time_vars() if args.push_results_redistimeseries: + logging.info( + f"Updating overall arch tests counter {ts_key_architecture}" + ) + rts.ts().add( + ts_key_architecture, + start_time_setup_ms, + 1, + duplicate_policy="sum", + ) logging.info( f"Updating overall spot price tests counter {ts_key_spot_price}" ) @@ -843,7 +855,10 @@ def run_remote_command_logic(args, project_name, project_version): tf_github_org, tf_github_repo, tf_triggering_env, - {"metric-type": "redis-metrics"}, + { + "metric-type": "redis-metrics", + "arch": architecture, + }, expire_ms, ) if collect_commandstats: @@ -866,7 +881,10 @@ def run_remote_command_logic(args, project_name, project_version): tf_github_org, tf_github_repo, tf_triggering_env, - {"metric-type": "commandstats"}, + { + "metric-type": "commandstats", + "arch": architecture, + }, expire_ms, ) ( @@ -888,7 +906,10 @@ def run_remote_command_logic(args, project_name, project_version): tf_github_org, tf_github_repo, tf_triggering_env, - {"metric-type": "latencystats"}, + { + "metric-type": "latencystats", + "arch": architecture, + }, expire_ms, ) except (