From 57e83053cbd1b61e1334cb2781ff4a0a6546576f Mon Sep 17 00:00:00 2001 From: minhkhul <118945681+minhkhul@users.noreply.github.com> Date: Mon, 10 Nov 2025 13:55:41 -0500 Subject: [PATCH 1/3] Add hsanci to nssp (#2162) * add hsanci code logic * lint: make happy * hsanci -> hsa-nci + add comment about min() for dedupe * use drop_duplicates * adjust test for new timeout --------- Co-authored-by: Dmitry Shemetov --- nssp/delphi_nssp/constants.py | 1 + nssp/delphi_nssp/pull.py | 4 ++-- nssp/delphi_nssp/run.py | 9 +++++++++ nssp/tests/test_pull.py | 2 +- 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/nssp/delphi_nssp/constants.py b/nssp/delphi_nssp/constants.py index fde27c3d0..ef7acb4f7 100644 --- a/nssp/delphi_nssp/constants.py +++ b/nssp/delphi_nssp/constants.py @@ -9,6 +9,7 @@ "state", "county", "hhs", + "hsa-nci", ] SIGNALS_MAP = { diff --git a/nssp/delphi_nssp/pull.py b/nssp/delphi_nssp/pull.py index 8113e418f..c18ee3961 100644 --- a/nssp/delphi_nssp/pull.py +++ b/nssp/delphi_nssp/pull.py @@ -113,7 +113,7 @@ def pull_with_socrata_api(socrata_token: str, dataset_id: str): ------- list of dictionaries, each representing a row in the dataset """ - client = Socrata("data.cdc.gov", socrata_token) + client = Socrata("data.cdc.gov", socrata_token, timeout=50) # set timeout to avoid read timed out error results = [] offset = 0 limit = 50000 # maximum limit allowed by SODA 2.0 @@ -177,5 +177,5 @@ def pull_nssp_data( # Format county fips to all be 5 digits with leading zeros df_ervisits["fips"] = df_ervisits["fips"].apply(lambda x: str(x).zfill(5) if str(x) != "0" else "0") - keep_columns = ["timestamp", "geography", "county", "fips"] + keep_columns = ["timestamp", "geography", "county", "fips", "hsa_nci_id"] return df_ervisits[SIGNALS + keep_columns] diff --git a/nssp/delphi_nssp/run.py b/nssp/delphi_nssp/run.py index d4e167d66..dc85055e5 100644 --- a/nssp/delphi_nssp/run.py +++ b/nssp/delphi_nssp/run.py @@ -45,6 +45,7 @@ def add_needed_columns(df, col_names=None): df = add_default_nancodes(df) return df + def logging(start_time, run_stats, logger): """Boilerplate making logs.""" elapsed_time_in_seconds = round(time.time() - start_time, 2) @@ -137,6 +138,14 @@ def run_module(params, logger=None): df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="geo_id") df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population") df = df.rename(columns={"weighted_val": "val"}) + elif geo == "hsa-nci": + df = df[["hsa_nci_id", "val", "timestamp"]] + df = df[df["hsa_nci_id"] != "All"] + # We use drop_duplicates below just to pick a representative value, + # since all the values in a given HSA-NCI level are the same + # (the data is reported at the HSA-NCI level). + df.drop_duplicates(["hsa_nci_id", "timestamp", "val"], inplace=True) + df = df.rename(columns={"hsa_nci_id": "geo_id"}) else: df = df[df["county"] != "All"] df["geo_id"] = df["fips"] diff --git a/nssp/tests/test_pull.py b/nssp/tests/test_pull.py index e70594528..c68c1906b 100644 --- a/nssp/tests/test_pull.py +++ b/nssp/tests/test_pull.py @@ -78,7 +78,7 @@ def test_normal_pull_nssp_data(self, mock_socrata, params, caplog): pd.testing.assert_frame_equal(expected_data, actual_data) # Check that Socrata client was initialized with correct arguments - mock_socrata.assert_called_once_with("data.cdc.gov", test_token) + mock_socrata.assert_called_once_with("data.cdc.gov", test_token, timeout=50) # Check that get method was called with correct arguments mock_client.get.assert_any_call("rdmq-nq56", limit=50000, offset=0) From b3c0ff96da06bdfb11b6e2f445cf85588495d53c Mon Sep 17 00:00:00 2001 From: Delphi Deploy Bot Date: Mon, 10 Nov 2025 19:24:39 +0000 Subject: [PATCH 2/3] chore: bump covidcast-indicators to 0.3.67 --- .bumpversion.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 525767ad9..015db3cd2 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 0.3.66 +current_version = 0.3.67 commit = True message = chore: bump covidcast-indicators to {new_version} tag = False From 8ffb216daadd4890fc0c68b5e7027db088633ddd Mon Sep 17 00:00:00 2001 From: minhkhul Date: Mon, 10 Nov 2025 19:24:40 +0000 Subject: [PATCH 3/3] [create-pull-request] automated change --- changehc/version.cfg | 2 +- claims_hosp/version.cfg | 2 +- doctor_visits/version.cfg | 2 +- google_symptoms/version.cfg | 2 +- hhs_hosp/version.cfg | 2 +- nchs_mortality/version.cfg | 2 +- nssp/version.cfg | 2 +- quidel_covidtest/version.cfg | 2 +- sir_complainsalot/version.cfg | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/changehc/version.cfg b/changehc/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/changehc/version.cfg +++ b/changehc/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/claims_hosp/version.cfg b/claims_hosp/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/claims_hosp/version.cfg +++ b/claims_hosp/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/doctor_visits/version.cfg b/doctor_visits/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/doctor_visits/version.cfg +++ b/doctor_visits/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/google_symptoms/version.cfg b/google_symptoms/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/google_symptoms/version.cfg +++ b/google_symptoms/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/hhs_hosp/version.cfg b/hhs_hosp/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/hhs_hosp/version.cfg +++ b/hhs_hosp/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/nchs_mortality/version.cfg b/nchs_mortality/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/nchs_mortality/version.cfg +++ b/nchs_mortality/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/nssp/version.cfg b/nssp/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/nssp/version.cfg +++ b/nssp/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/quidel_covidtest/version.cfg b/quidel_covidtest/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/quidel_covidtest/version.cfg +++ b/quidel_covidtest/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67 diff --git a/sir_complainsalot/version.cfg b/sir_complainsalot/version.cfg index c4ba8490d..02cc11ca2 100644 --- a/sir_complainsalot/version.cfg +++ b/sir_complainsalot/version.cfg @@ -1 +1 @@ -current_version = 0.3.66 +current_version = 0.3.67