clean state column (#2105)

* in progress

* filtering also on hhs and check in test

* regenerated test data to update current behavior

* lint

Author: aysim319

nhsn/delphi_nhsn/run.py

@@ -73,14 +73,17 @@ def run_module(params, logger=None):
                 df = df[df["geo_id"] == "us"]
             elif geo == "hhs":
                 df = df[df["geo_id"] != "us"]
+                df = df[df["geo_id"].str.len() == 2]
                 df.rename(columns={"geo_id": "state_id"}, inplace=True)
                 df = geo_mapper.add_geocode(df, "state_id", "state_code", from_col="state_id")
                 df = geo_mapper.add_geocode(df, "state_code", "hhs", from_col="state_code", new_col="hhs")
                 df = geo_mapper.replace_geocode(
                     df, from_col="state_code", from_code="state_code", new_col="geo_id", new_code="hhs"
                 )
-            else:
+            elif geo == "state":
                 df = df[df_pull["geo_id"] != "us"]
+                df = df[df["geo_id"].str.len() == 2]  # hhs region is a value in geo_id column
+
             df["se"] = np.nan
             df["sample_size"] = np.nan
             dates = create_export_csv(

nhsn/tests/test_data/20241212.csv.gz

@@ -2122,5 +2122,123 @@
     "pctconfc19icubedsperchos": "-0.0002",
     "pctconffluicubedsperchos": "-0.012",
     "pctconfrsvicubedsperchos": "0.0"
+  },
+  {
+    "weekendingdate": "2021-10-16T00:00:00.000",
+    "jurisdiction": "region 1",
+    "numinptbeds": "729772.05",
+    "numinptbedsadult": "660758.41",
+    "numinptbedsped": "53387.68",
+    "numinptbedsocc": "556817.29",
+    "numinptbedsoccadult": "510524.46",
+    "numinptbedsoccped": "35679.97",
+    "numicubeds": "114471.4",
+    "numicubedsadult": "86925.11",
+    "numicubedsped": "21902.45",
+    "numicubedsocc": "85344.84",
+    "numicubedsoccadult": "66877.69",
+    "numicubedsoccped": "14657.22",
+    "numconfc19hosppatsadult": "55448.05",
+    "numconfc19hosppatsped": "847.15",
+    "totalconfc19hosppats": "56295.2",
+    "totalconffluhosppats": "259.85",
+    "numconfc19icupatsadult": "18104.6",
+    "totalconfc19icupats": "18104.6",
+    "totalconffluicupats": "39.33",
+    "totalconfc19newadmped": "1301.0",
+    "numconfc19newadmadult18to49": "11532.0",
+    "totalconfc19newadmadult": "44677.0",
+    "numconfc19newadmunk": "1982.0",
+    "totalconfc19newadm": "45978.0",
+    "totalconfflunewadm": "266.0",
+    "pctinptbedsocc": "0.763",
+    "pctconfc19inptbeds": "0.0771",
+    "pctconffluinptbeds": "0.0004",
+    "pcticubedsocc": "0.7456",
+    "pctconfc19icubeds": "0.1582",
+    "pctconffluicubeds": "0.0003",
+    "pctconfc19newadmadult": "0.9717",
+    "pctconfc19newadmped": "0.0283",
+    "numinptbedshosprep": "5396",
+    "numinptbedsocchosprep": "5396",
+    "numicubedshosprep": "5396",
+    "numicubedsocchosprep": "5396",
+    "totalconfc19hosppatshosprep": "5396",
+    "totalconffluhosppatshosprep": "4317",
+    "totalconfrsvhosppatshosprep": "0",
+    "totalconfc19icupatshosprep": "5396",
+    "totalconffluicupatshosprep": "4306",
+    "totalconfrsvicupatshosprep": "0",
+    "totalconfc19newadmpedhosprep": "5278",
+    "totalconfc19newadmadulthosprep": "5394",
+    "totalconfc19newadmhosprep": "5394",
+    "totalconfflunewadmpedhosprep": "0",
+    "totalconfflunewadmadulthosprep": "0",
+    "totalconfflunewadmhosprep": "4307",
+    "totalconfrsvnewadmpedhosprep": "0",
+    "totalconfrsvnewadmadulthosprep": "0",
+    "totalconfrsvnewadmhosprep": "0",
+    "pctinptbedsocchosprep": "5396",
+    "pcticubedsocchosprep": "5396",
+    "pctconfc19inptbedshosprep": "5396",
+    "pctconffluinptbedshosprep": "4317",
+    "pctconfrsvinptbedshosprep": "0",
+    "pctconfc19icubedshosprep": "5396",
+    "pctconffluicubedshosprep": "4306",
+    "pctconfrsvicubedshosprep": "0",
+    "numinptbedsperchosprep": "0.9492",
+    "numinptbedsoccperchosprep": "0.9492",
+    "numicubedsperchosprep": "0.9492",
+    "numicubedsoccperchosprep": "0.9492",
+    "totalconfc19hosppatsperc": "0.9492",
+    "totalconffluhosppatsperc": "0.7594",
+    "totalconfrsvhosppatsperc": "0.0",
+    "totalconfc19icupatsperchosprep": "0.9492",
+    "totalconffluicupatsperchosprep": "0.7574",
+    "totalconfrsvicupatsperchosprep": "0.0",
+    "totalconfc19newadmpedper": "0.9284",
+    "totalconfc19newadmadultp": "0.9488",
+    "totalconfc19newadmperchosprep": "94.88",
+    "totalconfflunewadmpedper": "0.0",
+    "totalconfflunewadmadultp": "0.0",
+    "totalconfflunewadmperchosprep": "75.76",
+    "totalconfrsvnewadmpedper": "0.0",
+    "totalconfrsvnewadmadultp": "0.0",
+    "totalconfrsvnewadmperchosprep": "0.0",
+    "pctinptbedsoccperchosprep": "0.9492",
+    "pcticubedsoccperchosprep": "0.9492",
+    "pctconfc19inptbedsperchosprep": "0.9492",
+    "pctconffluinptbedsperchosprep": "0.7594",
+    "pctconfrsvinptbedsperchosprep": "0.0",
+    "pctconfc19icubedsperchosprep": "0.9492",
+    "pctconffluicubedsperchosprep": "0.7574",
+    "pctconfrsvicubedsperchosprep": "0.0",
+    "numinptbedsperchosprepabschg": "-0.07",
+    "numinptbedsoccperchospre": "-0.07",
+    "numicubedsperchosprepabschg": "-0.07",
+    "numicubedsoccperchosprepabschg": "-0.07",
+    "totalconfc19hosppatsperc_1": "-0.02",
+    "totalconffluhosppatsperc_1": "-1.23",
+    "totalconfrsvhosppatsperc_1": "0.0",
+    "totalconfc19icupatsperch": "-0.02",
+    "totalconffluicupatsperch": "-1.2",
+    "totalconfrsvicupatsperch": "0.0",
+    "totalconfc19newadmpedper_1": "-0.04",
+    "totalconfc19newadmadultp_1": "-0.04",
+    "totalconfc19newadmpercho": "-0.04",
+    "totalconfflunewadmpedper_1": "0.0",
+    "totalconfflunewadmadultp_1": "0.0",
+    "totalconfflunewadmpercho": "-1.27",
+    "totalconfrsvnewadmpedper_1": "0.0",
+    "totalconfrsvnewadmadultp_1": "0.0",
+    "totalconfrsvnewadmpercho": "0.0",
+    "pctinptbedsoccperchospre": "-0.0007",
+    "pcticubedsoccperchosprepabschg": "-0.0007",
+    "pctconfc19inptbedspercho": "-0.0002",
+    "pctconffluinptbedspercho": "-0.0123",
+    "pctconfrsvinptbedspercho": "0.0",
+    "pctconfc19icubedsperchos": "-0.0002",
+    "pctconffluicubedsperchos": "-0.012",
+    "pctconfrsvicubedsperchos": "0.0"
   }
 ]

nhsn/tests/test_data/page.json

@@ -45,6 +45,9 @@ def test_output_files_exist(self, params, run_as_module):
                 "geo_id", "val", "se", "sample_size",
             ]
             assert (df.columns.values == expected_columns).all()
+            if geo == "state":
+                states = list(df["geo_id"].values)
+                assert all(len(state) == 2 for state in states)
 
         for file in Path(export_dir).glob("*.csv"):
             os.remove(file)