Merge pull request #2165 from cmu-delphi/release/indicators_v0.3.67_utils_v0.3.27

Release covidcast-indicators 0.3.67

Author: melange396

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.3.66
+current_version = 0.3.67
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

changehc/version.cfg

@@ -1 +1 @@
-current_version = 0.3.66
+current_version = 0.3.67

claims_hosp/version.cfg

@@ -1 +1 @@
-current_version = 0.3.66
+current_version = 0.3.67

doctor_visits/version.cfg

@@ -1 +1 @@
-current_version = 0.3.66
+current_version = 0.3.67

google_symptoms/version.cfg

@@ -1 +1 @@
-current_version = 0.3.66
+current_version = 0.3.67

hhs_hosp/version.cfg

@@ -1 +1 @@
-current_version = 0.3.66
+current_version = 0.3.67

nchs_mortality/version.cfg

@@ -1 +1 @@
-current_version = 0.3.66
+current_version = 0.3.67

nssp/delphi_nssp/constants.py

@@ -9,6 +9,7 @@
     "state",
     "county",
     "hhs",
+    "hsa_nci",
 ]
 
 SIGNALS_MAP = {

nssp/delphi_nssp/pull.py

@@ -113,7 +113,7 @@ def pull_with_socrata_api(socrata_token: str, dataset_id: str):
     -------
     list of dictionaries, each representing a row in the dataset
     """
-    client = Socrata("data.cdc.gov", socrata_token)
+    client = Socrata("data.cdc.gov", socrata_token, timeout=50)  # set timeout to avoid read timed out error
     results = []
     offset = 0
     limit = 50000  # maximum limit allowed by SODA 2.0
@@ -177,5 +177,5 @@ def pull_nssp_data(
     # Format county fips to all be 5 digits with leading zeros
     df_ervisits["fips"] = df_ervisits["fips"].apply(lambda x: str(x).zfill(5) if str(x) != "0" else "0")
 
-    keep_columns = ["timestamp", "geography", "county", "fips"]
+    keep_columns = ["timestamp", "geography", "county", "fips", "hsa_nci_id"]
     return df_ervisits[SIGNALS + keep_columns]

nssp/delphi_nssp/run.py

@@ -45,6 +45,7 @@ def add_needed_columns(df, col_names=None):
     df = add_default_nancodes(df)
     return df
 
+
 def logging(start_time, run_stats, logger):
     """Boilerplate making logs."""
     elapsed_time_in_seconds = round(time.time() - start_time, 2)
@@ -137,6 +138,14 @@ def run_module(params, logger=None):
                 df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="geo_id")
                 df = geo_mapper.aggregate_by_weighted_sum(df, "geo_id", "val", "timestamp", "population")
                 df = df.rename(columns={"weighted_val": "val"})
+            elif geo == "hsa_nci":
+                df = df[["hsa_nci_id", "val", "timestamp"]]
+                df = df[df["hsa_nci_id"] != "All"]
+                # We use drop_duplicates below just to pick a representative value,
+                # since all the values in a given HSA-NCI level are the same
+                # (the data is reported at the HSA-NCI level).
+                df.drop_duplicates(["hsa_nci_id", "timestamp", "val"], inplace=True)
+                df = df.rename(columns={"hsa_nci_id": "geo_id"})
             else:
                 df = df[df["county"] != "All"]
                 df["geo_id"] = df["fips"]