From a1c2f61cba1fc967a2d1eb13171d0de9a3656969 Mon Sep 17 00:00:00 2001 From: minhkhul Date: Fri, 7 Nov 2025 17:35:12 -0500 Subject: [PATCH 1/5] add hsanci code logic --- nssp/delphi_nssp/constants.py | 1 + nssp/delphi_nssp/pull.py | 2 +- nssp/delphi_nssp/run.py | 5 +++++ 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/nssp/delphi_nssp/constants.py b/nssp/delphi_nssp/constants.py index fde27c3d0..806b5e8cb 100644 --- a/nssp/delphi_nssp/constants.py +++ b/nssp/delphi_nssp/constants.py @@ -9,6 +9,7 @@ "state", "county", "hhs", + "hsanci", ] SIGNALS_MAP = { diff --git a/nssp/delphi_nssp/pull.py b/nssp/delphi_nssp/pull.py index 8113e418f..c6e318958 100644 --- a/nssp/delphi_nssp/pull.py +++ b/nssp/delphi_nssp/pull.py @@ -177,5 +177,5 @@ def pull_nssp_data( # Format county fips to all be 5 digits with leading zeros df_ervisits["fips"] = df_ervisits["fips"].apply(lambda x: str(x).zfill(5) if str(x) != "0" else "0") - keep_columns = ["timestamp", "geography", "county", "fips"] + keep_columns = ["timestamp", "geography", "county", "fips", "hsa_nci_id"] return df_ervisits[SIGNALS + keep_columns] diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py index d4e167d66..23d6488d4 100644 --- a/nssp/delphi_nssp/run.py +++ b/nssp/delphi_nssp/run.py @@ -137,6 +137,11 @@ def run_module(params, logger=None): df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="geo_id") df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population") df = df.rename(columns={"weighted_val": "val"}) + elif geo == "hsanci": + df = df[["hsa_nci_id", "val", "timestamp"]] + df = df[df["hsa_nci_id"] != "All"] + df = df.groupby(["hsa_nci_id", "timestamp"])['val'].min().reset_index() + df = df.rename(columns={"hsa_nci_id": "geo_id"}) else: df = df[df["county"] != "All"] df["geo_id"] = df["fips"] From 80bb904cf15defe06d895fa8779e16103ae47e4e Mon Sep 17 00:00:00 2001 From: Dmitry Shemetov Date: Fri, 7 Nov 2025 14:59:09 -0800 Subject: [PATCH 2/5] lint: make happy --- nssp/delphi_nssp/run.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py index 23d6488d4..3cd3eed51 100644 --- a/nssp/delphi_nssp/run.py +++ b/nssp/delphi_nssp/run.py @@ -45,6 +45,7 @@ def add_needed_columns(df, col_names=None): df = add_default_nancodes(df) return df + def logging(start_time, run_stats, logger): """Boilerplate making logs.""" elapsed_time_in_seconds = round(time.time() - start_time, 2) @@ -140,7 +141,7 @@ def run_module(params, logger=None): elif geo == "hsanci": df = df[["hsa_nci_id", "val", "timestamp"]] df = df[df["hsa_nci_id"] != "All"] - df = df.groupby(["hsa_nci_id", "timestamp"])['val'].min().reset_index() + df = df.groupby(["hsa_nci_id", "timestamp"])["val"].min().reset_index() df = df.rename(columns={"hsa_nci_id": "geo_id"}) else: df = df[df["county"] != "All"] From c5d5dccda59416d23da5306c0c886e483bd22d5c Mon Sep 17 00:00:00 2001 From: minhkhul Date: Fri, 7 Nov 2025 18:21:41 -0500 Subject: [PATCH 3/5] hsanci -> hsa-nci + add comment about min() for dedupe --- nssp/delphi_nssp/constants.py | 2 +- nssp/delphi_nssp/run.py | 7 +++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/nssp/delphi_nssp/constants.py b/nssp/delphi_nssp/constants.py index 806b5e8cb..ef7acb4f7 100644 --- a/nssp/delphi_nssp/constants.py +++ b/nssp/delphi_nssp/constants.py @@ -9,7 +9,7 @@ "state", "county", "hhs", - "hsanci", + "hsa-nci", ] SIGNALS_MAP = { diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py index 23d6488d4..edef7a3e2 100644 --- a/nssp/delphi_nssp/run.py +++ b/nssp/delphi_nssp/run.py @@ -137,10 +137,13 @@ def run_module(params, logger=None): df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="geo_id") df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population") df = df.rename(columns={"weighted_val": "val"}) - elif geo == "hsanci": + elif geo == "hsa-nci": df = df[["hsa_nci_id", "val", "timestamp"]] df = df[df["hsa_nci_id"] != "All"] - df = df.groupby(["hsa_nci_id", "timestamp"])['val'].min().reset_index() + # We use min() below just to pick a representative value, since all + # the values in a given HSA-NCI level are the same (the data is + # reported at the HSA-NCI level). + df = df.groupby(["hsa_nci_id", "timestamp"])["val"].min().reset_index() df = df.rename(columns={"hsa_nci_id": "geo_id"}) else: df = df[df["county"] != "All"] From 615572d952a02c2a949452829a0853f6bfe7a180 Mon Sep 17 00:00:00 2001 From: minhkhul Date: Mon, 10 Nov 2025 13:19:01 -0500 Subject: [PATCH 4/5] use drop_duplicates --- nssp/delphi_nssp/pull.py | 2 +- nssp/delphi_nssp/run.py | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nssp/delphi_nssp/pull.py b/nssp/delphi_nssp/pull.py index c6e318958..c18ee3961 100644 --- a/nssp/delphi_nssp/pull.py +++ b/nssp/delphi_nssp/pull.py @@ -113,7 +113,7 @@ def pull_with_socrata_api(socrata_token: str, dataset_id: str): ------- list of dictionaries, each representing a row in the dataset """ - client = Socrata("data.cdc.gov", socrata_token) + client = Socrata("data.cdc.gov", socrata_token, timeout=50) # set timeout to avoid read timed out error results = [] offset = 0 limit = 50000 # maximum limit allowed by SODA 2.0 diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py index 6b971612e..dc85055e5 100644 --- a/nssp/delphi_nssp/run.py +++ b/nssp/delphi_nssp/run.py @@ -141,10 +141,10 @@ def run_module(params, logger=None): elif geo == "hsa-nci": df = df[["hsa_nci_id", "val", "timestamp"]] df = df[df["hsa_nci_id"] != "All"] - # We use min() below just to pick a representative value, since all - # the values in a given HSA-NCI level are the same (the data is - # reported at the HSA-NCI level). - df = df.groupby(["hsa_nci_id", "timestamp"])["val"].min().reset_index() + # We use drop_duplicates below just to pick a representative value, + # since all the values in a given HSA-NCI level are the same + # (the data is reported at the HSA-NCI level). + df.drop_duplicates(["hsa_nci_id", "timestamp", "val"], inplace=True) df = df.rename(columns={"hsa_nci_id": "geo_id"}) else: df = df[df["county"] != "All"] From d4b2642aeb3da48cec9328aab99be0f9ab337a6f Mon Sep 17 00:00:00 2001 From: minhkhul Date: Mon, 10 Nov 2025 13:46:04 -0500 Subject: [PATCH 5/5] adjust test for new timeout --- nssp/tests/test_pull.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nssp/tests/test_pull.py b/nssp/tests/test_pull.py index e70594528..c68c1906b 100644 --- a/nssp/tests/test_pull.py +++ b/nssp/tests/test_pull.py @@ -78,7 +78,7 @@ def test_normal_pull_nssp_data(self, mock_socrata, params, caplog): pd.testing.assert_frame_equal(expected_data, actual_data) # Check that Socrata client was initialized with correct arguments - mock_socrata.assert_called_once_with("data.cdc.gov", test_token) + mock_socrata.assert_called_once_with("data.cdc.gov", test_token, timeout=50) # Check that get method was called with correct arguments mock_client.get.assert_any_call("rdmq-nq56", limit=50000, offset=0)