Merge pull request #1498 from cmu-delphi/release/indicators_v0.2.24_utils_v0.2.11

Release covidcast-indicators 0.2.24

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.23
+current_version = 0.2.24
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

ansible/templates/google_symptoms-params-prod.json.j2

@@ -26,6 +26,12 @@
       "min_expected_lag": {"all": "3"},
       "max_expected_lag": {"all": "4"},
       "suppressed_errors": [
+        {"signal": "ageusia_raw_search"},
+        {"signal": "ageusia_smoothed_search"},
+        {"signal": "anosmia_raw_search"},
+        {"signal": "anosmia_smoothed_search"},
+        {"signal": "sum_anosmia_ageusia_raw_search"},
+        {"signal": "sum_anosmia_ageusia_smoothed_search"}
       ]
     },
     "static": {
@@ -39,9 +45,9 @@
         "s01_smoothed_search",
         "s02_smoothed_search",
         "s03_smoothed_search",
+        "s04_smoothed_search",
         "s05_smoothed_search",
         "s06_smoothed_search",
-        "s08_smoothed_search",
         "scontrol_smoothed_search"
       ]
     }

google_symptoms/delphi_google_symptoms/constants.py

@@ -11,11 +11,11 @@
             "Rhinitis", "Common cold"],
     "s03": ["Fever", "Hyperthermia", "Chills", "Shivering", "Low grade fever"],
     #"s04": ["Fatigue", "Weakness", "Muscle weakness", "Myalgia", "Pain"],
-    "s05": ["Shortness of breath", "Wheeze", "Croup", "Pneumonia", "Asthma",
+    "s04": ["Shortness of breath", "Wheeze", "Croup", "Pneumonia", "Asthma",
             "Crackles", "Acute bronchitis", "Bronchitis"],
-    "s06": ["Anosmia", "Dysgeusia", "Ageusia"],
+    "s05": ["Anosmia", "Dysgeusia", "Ageusia"],
     #"s07": ["Nausea", "Vomiting", "Diarrhea", "Indigestion", "Abdominal pain"],
-    "s08": ["Laryngitis", "Sore throat", "Throat irritation"],
+    "s06": ["Laryngitis", "Sore throat", "Throat irritation"],
     #"s09": ["Headache", "Migraine", "Cluster headache", "Dizziness", "Lightheadedness"],
     #"s10": ["Night sweats","Perspiration", "hyperhidrosis"],
     "scontrol": ["Type 2 diabetes", "Urinary tract infection", "Hair loss",

google_symptoms/delphi_google_symptoms/run.py

@@ -23,6 +23,8 @@
 from .pull import pull_gs_data
 
 
+# pylint: disable=R0912
+# pylint: disable=R0915
 def run_module(params):
     """
     Run Google Symptoms module.
@@ -57,22 +59,37 @@ def run_module(params):
     num_export_days = params["indicator"]["num_export_days"]
 
     if num_export_days is None:
-        # Calculate number of days based on what's missing from the API.
+        # Generate a list of signals we expect to produce
+        sensor_names = set(
+            "_".join([metric, smoother, "search"])
+            for metric, smoother in product(COMBINED_METRIC, SMOOTHERS)
+        )
+
+        # Fetch metadata to check how recent each signal is
         metadata = covidcast.metadata()
-        gs_metadata = metadata[(metadata.data_source == "google-symptoms")]
-
-        # Calculate number of days based on what validator expects.
-        max_expected_lag = lag_converter(
-            params["validation"]["common"].get("max_expected_lag", {"all": 4})
-            )
-        global_max_expected_lag = max( list(max_expected_lag.values()) )
-
-        # Select the larger number of days. Prevents validator from complaining about missing dates,
-        # and backfills in case of an outage.
-        num_export_days = max(
-            (datetime.today() - to_datetime(min(gs_metadata.max_time))).days + 1,
-            params["validation"]["common"].get("span_length", 14) + global_max_expected_lag
-            )
+        # Filter to only those we currently want to produce, ignore any old or deprecated signals
+        gs_metadata = metadata[(metadata.data_source == "google-symptoms") &
+            (metadata.signal.isin(sensor_names))]
+
+        if sensor_names.difference(set(gs_metadata.signal)):
+            # If any signal not in metadata yet, we need to backfill its full history.
+            num_export_days = "all"
+        else:
+            # Calculate number of days based on what's missing from the API and
+            # what the validator expects.
+            max_expected_lag = lag_converter(
+                params["validation"]["common"].get("max_expected_lag", {"all": 4})
+                )
+            global_max_expected_lag = max( list(max_expected_lag.values()) )
+
+            # Select the larger number of days. Prevents validator from complaining about
+            # missing dates, and backfills in case of an outage.
+            num_export_days = max(
+                (datetime.today() - to_datetime(min(gs_metadata.max_time))).days + 1,
+                params["validation"]["common"].get("span_length", 14) + global_max_expected_lag
+                )
+    if num_export_days == "all":
+        num_export_days = (export_end_date - export_start_date).days + 1
 
     logger = get_structured_logger(
         __name__, filename=params["common"].get("log_filename"),

google_symptoms/params.json.template

@@ -16,6 +16,12 @@
       "min_expected_lag": {"all": "3"},
       "max_expected_lag": {"all": "4"},
       "suppressed_errors": [
+        {"signal": "ageusia_raw_search"},
+        {"signal": "ageusia_smoothed_search"},
+        {"signal": "anosmia_raw_search"},
+        {"signal": "anosmia_smoothed_search"},
+        {"signal": "sum_anosmia_ageusia_raw_search"},
+        {"signal": "sum_anosmia_ageusia_smoothed_search"}
       ]
     },
     "static": {
@@ -29,9 +35,9 @@
         "s01_smoothed_search",
         "s02_smoothed_search",
         "s03_smoothed_search",
+        "s04_smoothed_search",
         "s05_smoothed_search",
         "s06_smoothed_search",
-        "s08_smoothed_search",
         "scontrol_smoothed_search"
       ]
     }

google_symptoms/setup.py

@@ -4,7 +4,7 @@
 required = [
     "mock",
     "numpy",
-    "pandas",
+    "pandas==1.3.5",
     "pydocstyle",
     "pytest",
     "pytest-cov",

google_symptoms/tests/test_run.py

@@ -25,9 +25,9 @@ def test_output_files_exist(self, run_as_module):
         geos = ["county", "state", "hhs", "nation"]
         metrics = ["s01", "s02", "s03",
                    #"s04",
-                   "s05", "s06",
+                   "s04", "s05",
                    #"s07",
-                   "s08",
+                   "s06",
                    #"s09", "s10",
                    "scontrol"]
         smoother = ["raw", "smoothed"]

safegraph_patterns/setup.py

@@ -7,6 +7,7 @@
     "pydocstyle",
     "pytest",
     "pytest-cov",
+    "coverage==6.2.0",
     "pylint==2.8.3",
     "delphi-utils"
 ]