Merge pull request #1481 from cmu-delphi/release/indicators_v0.2.22_utils_v0.2.10

Release covidcast-indicators 0.2.22

Author: krivard

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.21
+current_version = 0.2.22
 commit = True
 message = chore: bump covidcast-indicators to {new_version}
 tag = False

.github/workflows/python-ci.yml

@@ -16,7 +16,7 @@ jobs:
     if: github.event.pull_request.draft == false
     strategy:
       matrix:
-        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts]
+        packages: [_delphi_utils_python, changehc, claims_hosp, combo_cases_and_deaths, doctor_visits, dsew_community_profile, google_symptoms, hhs_hosp, hhs_facilities, jhu, nchs_mortality, nowcast, quidel, quidel_covidtest, safegraph_patterns, sir_complainsalot, usafacts]
     defaults:
       run:
         working-directory: ${{ matrix.packages }}

Jenkinsfile

@@ -9,7 +9,7 @@
    - Keep in sync with '.github/workflows/python-ci.yml'.
    - TODO: #527 Get this list automatically from python-ci.yml at runtime.
  */
-def indicator_list = ["changehc", "claims_hosp", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts"]
+def indicator_list = ["changehc", "claims_hosp", "facebook", "google_symptoms", "hhs_hosp", "jhu", "nchs_mortality", "quidel", "quidel_covidtest", "safegraph_patterns", "sir_complainsalot", "usafacts", "dsew_community_profile"]
 def build_package = [:]
 def deploy_staging = [:]
 def deploy_production = [:]

_delphi_utils_python/.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.2.9
+current_version = 0.2.10
 commit = True
 message = chore: bump delphi_utils to {new_version}
 tag = False

_delphi_utils_python/delphi_utils/__init__.py

@@ -15,4 +15,4 @@
 from .nancodes import Nans
 from .weekday import Weekday
 
-__version__ = "0.2.9"
+__version__ = "0.2.10"

_delphi_utils_python/delphi_utils/validator/datafetcher.py

@@ -111,14 +111,9 @@ def get_geo_signal_combos(data_source):
     Cross references based on combinations reported available by COVIDcast metadata.
     """
     # Maps data_source name with what's in the API, lists used in case of multiple names
-    # pylint: disable=fixme
-    # TODO: Extract this mapping from meta response instead of hard-coding
-    # https://github.com/cmu-delphi/covidcast-indicators/issues/1457
-    source_signal_mappings = {
-        'indicator-combination': ['indicator-combination-cases-deaths'],
-        'quidel': ['quidel-covid-ag'],
-        'safegraph': ['safegraph-weekly']
-    }
+
+    source_signal_mappings = {i['source']:i['db_source'] for i in
+        requests.get("https://api.covidcast.cmu.edu/epidata/covidcast/meta").json()}
     meta = covidcast.metadata()
     source_meta = meta[meta['data_source'] == data_source]
     # Need to convert np.records to tuples so they are hashable and can be used in sets and dicts.
@@ -130,8 +125,9 @@ def get_geo_signal_combos(data_source):
     # True/False indicate if status is active, "unknown" means we should check
     sig_combo_seen = dict()
     for combo in geo_signal_combos:
-        if source_signal_mappings.get(data_source):
-            src_list = source_signal_mappings.get(data_source)
+        if data_source in source_signal_mappings.values():
+            src_list = [key for (key, value) in source_signal_mappings.items()
+                if value == data_source]
         else:
             src_list = [data_source]
         for src in src_list:

_delphi_utils_python/setup.py

@@ -26,7 +26,7 @@
 
 setup(
     name="delphi_utils",
-    version="0.2.9",
+    version="0.2.10",
     description="Shared Utility Functions for Indicators",
     long_description=long_description,
     long_description_content_type="text/markdown",

_delphi_utils_python/tests/validator/test_datafetcher.py

@@ -21,25 +21,44 @@ def test_make_date_filter(self):
         assert not date_filter(FILENAME_REGEX.match("20200620_a_b.csv"))
         assert not date_filter(FILENAME_REGEX.match("202006_a_b.csv"))
 
-    # pylint: disable=fixme
-    # TODO: mock out the advanced meta endpoint /covidcast/meta as well
-    # https://github.com/cmu-delphi/covidcast-indicators/issues/1456
+    # Solution from https://stackoverflow.com/questions/15753390/
+    #how-can-i-mock-requests-and-the-response
+    def mocked_requests_get(*args, **kwargs):
+        class MockResponse:
+            def __init__(self, json_data, status_code):
+                self.json_data = json_data
+                self.status_code = status_code
+
+            def json(self):
+                return self.json_data
+        if len(kwargs) == 0:
+            return MockResponse([{'source': 'chng', 'db_source': 'chng'},
+                {'source': 'covid-act-now', 'db_source': 'covid-act-now'}], 200)
+        elif kwargs["params"] == {'signal': 'chng:inactive'}:
+            return MockResponse([{"signals": [{"active": False}]}], 200)
+        else:
+            return MockResponse([{"signals": [{"active": True}]}], 200)
+    @mock.patch('requests.get', side_effect=mocked_requests_get)
     @mock.patch("covidcast.metadata")
-    def test_get_geo_signal_combos(self, mock_metadata):
+    def test_get_geo_signal_combos(self, mock_metadata, mock_get):
         """Test that the geo signal combos are correctly pulled from the covidcast metadata."""
         # Need to use actual data_source and signal names since we reference the API
+        # We let the chng signal "inactive" be an inactive signal
         mock_metadata.return_value = pd.DataFrame({"data_source": ["chng", "chng", "chng",
                                                                    "covid-act-now",
                                                                    "covid-act-now",
-                                                                   "covid-act-now"],
+                                                                   "covid-act-now",
+                                                                   "chng"],
                                                    "signal": ["smoothed_outpatient_cli",
                                                               "smoothed_outpatient_covid",
                                                               "smoothed_outpatient_covid",
                                                               "pcr_specimen_positivity_rate",
                                                               "pcr_specimen_positivity_rate",
-                                                              "pcr_specimen_total_tests"],
+                                                              "pcr_specimen_total_tests",
+                                                              "inactive"],
                                                    "geo_type": ["state", "state", "county",
-                                                                "hrr", "msa", "msa"]
+                                                                "hrr", "msa", "msa",
+                                                                "state"]
                                                   })
 
         assert set(get_geo_signal_combos("chng")) == set(

ansible/templates/dsew_community_profile-params-prod.json.j2

@@ -0,0 +1,32 @@
+{
+  "common": {
+    "export_dir": "/common/covidcast/receiving/dsew-cpr",
+    "log_filename": "/var/log/indicators/dsew_cpr.log"
+  },
+  "indicator": {
+    "input_cache": "./input_cache",
+    "reports": "new"
+  },
+  "validation": {
+    "common": {
+      "data_source": "dsew-cpr",
+      "span_length": 14,
+      "min_expected_lag": {"all": "5"},
+      "max_expected_lag": {"all": "9"},
+      "dry_run": true,
+      "suppressed_errors": []
+    },
+    "static": {
+      "minimum_sample_size": 0,
+      "missing_se_allowed": true,
+      "missing_sample_size_allowed": true
+    },
+    "dynamic": {
+      "ref_window_size": 7,
+      "smoothed_signals": [
+        "naats_total_7dav",
+        "naats_positivity_7dav"
+      ]
+    }
+  }
+}

ansible/templates/facebook-params-prod.json.j2

@@ -38,6 +38,7 @@
         "Survey of COVID-Like Illness - Wave 11": "fb-survey",
         "Survey of COVID-Like Illness - Wave 12": "fb-survey",
         "Survey of COVID-Like Illness - Wave 12 - Full Launch": "fb-survey",
+        "Survey of COVID-Like Illness - Wave 13": "fb-survey",
         "Survey of COVID-Like Illness - Wave 4": "fb-survey",
         "Survey of COVID-Like Illness - Wave 5": "fb-survey",
         "Survey of COVID-Like Illness - Wave 6": "fb-survey",