From 31f2fe8cb7d5f1f0d9b0c23a2287b4d68c9ffc11 Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 6 Nov 2025 12:08:02 -0500 Subject: [PATCH 1/4] catch and reissue imputation errors as warnings to avoid failures --- src/acquisition/fluview/impute_missing_values.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/acquisition/fluview/impute_missing_values.py b/src/acquisition/fluview/impute_missing_values.py index c795d9cce..5a5e8f3c9 100644 --- a/src/acquisition/fluview/impute_missing_values.py +++ b/src/acquisition/fluview/impute_missing_values.py @@ -51,6 +51,7 @@ # third party import mysql.connector import numpy as np +from warnings import warn # first party import delphi.operations.secrets as secrets @@ -58,6 +59,8 @@ from delphi.utils.geo.locations import Locations +UNDERDETERMINED_MSG = "system is underdetermined" + class Database: """Database wrapper and abstraction layer.""" @@ -240,7 +243,7 @@ def get_fusion_parameters(known_locations): H = graph[is_known, :] W = graph[is_unknown, :] if np.linalg.matrix_rank(H) != len(atoms): - raise StatespaceException("system is underdetermined") + raise StatespaceException(UNDERDETERMINED_MSG) HtH = np.dot(H.T, H) HtH_inv = np.linalg.inv(HtH) @@ -293,7 +296,15 @@ def impute_missing_values(database, test_mode=False): known_values["pr"] = (0, 0, 0) # get the imputation matrix and lists of known and unknown locations - F, known, unknown = get_fusion_parameters(known_values.keys()) + try: + F, known, unknown = get_fusion_parameters(known_values.keys()) + except StatespaceException as e: + message = str(e) + if message == UNDERDETERMINED_MSG: + warn(message) + else: + print(message) + continue # finally, impute the missing values z = np.array([known_values[k] for k in known]) From 249685756ed6024dc3acc8c65d521f6b87f29cbf Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 6 Nov 2025 12:25:38 -0500 Subject: [PATCH 2/4] tests to look for warnings instead --- tests/acquisition/fluview/test_impute_missing_values.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/acquisition/fluview/test_impute_missing_values.py b/tests/acquisition/fluview/test_impute_missing_values.py index 96338d467..db494346d 100644 --- a/tests/acquisition/fluview/test_impute_missing_values.py +++ b/tests/acquisition/fluview/test_impute_missing_values.py @@ -11,7 +11,6 @@ get_argument_parser, get_lag_and_ili, impute_missing_values, - StatespaceException, ) # py3tester coverage target @@ -87,7 +86,7 @@ def test_impute_missing_values_vipr(self): db.get_known_values.return_value = known_data db.find_missing_rows.return_value = [(201340, 201340)] - with self.assertRaises(Exception): + with self.assertWarns(Warning): impute_missing_values(db, test_mode=True) db.find_missing_rows.return_value = [(201339, 201339)] @@ -131,5 +130,5 @@ def test_impute_missing_values_underdetermined(self): db.find_missing_rows.return_value = [(201740, 201740)] db.get_known_values.return_value = known_data - with self.assertRaises(StatespaceException): + with self.assertWarns(Warning): impute_missing_values(db, test_mode=True) From 3d12e8325d82b62df0203254948c9245b58234e7 Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 6 Nov 2025 17:33:56 -0500 Subject: [PATCH 3/4] warn() goes to stderr instead of stdout, so print all messages --- .../fluview/impute_missing_values.py | 10 ++-------- .../fluview/test_impute_missing_values.py | 17 +++++++++++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/acquisition/fluview/impute_missing_values.py b/src/acquisition/fluview/impute_missing_values.py index 5a5e8f3c9..eaff369fb 100644 --- a/src/acquisition/fluview/impute_missing_values.py +++ b/src/acquisition/fluview/impute_missing_values.py @@ -59,8 +59,6 @@ from delphi.utils.geo.locations import Locations -UNDERDETERMINED_MSG = "system is underdetermined" - class Database: """Database wrapper and abstraction layer.""" @@ -243,7 +241,7 @@ def get_fusion_parameters(known_locations): H = graph[is_known, :] W = graph[is_unknown, :] if np.linalg.matrix_rank(H) != len(atoms): - raise StatespaceException(UNDERDETERMINED_MSG) + raise StatespaceException("system is underdetermined") HtH = np.dot(H.T, H) HtH_inv = np.linalg.inv(HtH) @@ -299,11 +297,7 @@ def impute_missing_values(database, test_mode=False): try: F, known, unknown = get_fusion_parameters(known_values.keys()) except StatespaceException as e: - message = str(e) - if message == UNDERDETERMINED_MSG: - warn(message) - else: - print(message) + print(e) continue # finally, impute the missing values diff --git a/tests/acquisition/fluview/test_impute_missing_values.py b/tests/acquisition/fluview/test_impute_missing_values.py index db494346d..328d262a1 100644 --- a/tests/acquisition/fluview/test_impute_missing_values.py +++ b/tests/acquisition/fluview/test_impute_missing_values.py @@ -2,8 +2,10 @@ # standard library import argparse +import sys import unittest from unittest.mock import MagicMock +from io import StringIO # first party from delphi.utils.geo.locations import Locations @@ -86,8 +88,12 @@ def test_impute_missing_values_vipr(self): db.get_known_values.return_value = known_data db.find_missing_rows.return_value = [(201340, 201340)] - with self.assertWarns(Warning): - impute_missing_values(db, test_mode=True) + + capturedOutput = StringIO() + sys.stdout = capturedOutput + impute_missing_values(db, test_mode=True) + sys.stdout = sys.__stdout__ + self.assertTrue("system is underdetermined" in capturedOutput.getvalue().split("\n")) db.find_missing_rows.return_value = [(201339, 201339)] impute_missing_values(db, test_mode=True) @@ -130,5 +136,8 @@ def test_impute_missing_values_underdetermined(self): db.find_missing_rows.return_value = [(201740, 201740)] db.get_known_values.return_value = known_data - with self.assertWarns(Warning): - impute_missing_values(db, test_mode=True) + capturedOutput = StringIO() + sys.stdout = capturedOutput + impute_missing_values(db, test_mode=True) + sys.stdout = sys.__stdout__ + self.assertTrue("system is underdetermined" in capturedOutput.getvalue().split("\n")) From d5efbf0d0d341c440a9dde989070c8235482e06d Mon Sep 17 00:00:00 2001 From: nmdefries <42820733+nmdefries@users.noreply.github.com> Date: Thu, 6 Nov 2025 17:35:26 -0500 Subject: [PATCH 4/4] drop warn import --- src/acquisition/fluview/impute_missing_values.py | 1 - 1 file changed, 1 deletion(-) diff --git a/src/acquisition/fluview/impute_missing_values.py b/src/acquisition/fluview/impute_missing_values.py index eaff369fb..3fc4b06e8 100644 --- a/src/acquisition/fluview/impute_missing_values.py +++ b/src/acquisition/fluview/impute_missing_values.py @@ -51,7 +51,6 @@ # third party import mysql.connector import numpy as np -from warnings import warn # first party import delphi.operations.secrets as secrets