Add a custom_validate_statistics function to the TFDV validation API.

PiperOrigin-RevId: 488706802

Author: caveness

.bazelrc

@@ -0,0 +1,13 @@
+# Needed to work with ZetaSQL dependency.
+build --cxxopt="-std=c++17"
+
+# Due to the invalid escape sequence in rules_foreign_cc
+# (e.g. "\(" in windows_commands.bzl) and the bazel 4.0.0 updates
+# (https://github.com/bazelbuild/bazel/commit/73402fa4aa5b9de46c9a4042b75e6fb332ad4a7f).
+build --incompatible_restrict_string_escapes=false
+
+# icu@: In create_linking_context: in call to create_linking_context(),
+# parameter 'user_link_flags' is deprecated and will be removed soon.
+# It may be temporarily re-enabled by setting --incompatible_require_linker_input_cc_api=false
+build --incompatible_require_linker_input_cc_api=false
+

RELEASE.md

@@ -9,6 +9,9 @@
 
 ## Major Features and Improvements
 
+*  Add a `custom_validate_statistics` function to the validation API. Note that
+   this function is not available on Windows.
+
 ## Bug Fixes and Other Changes
 
 *   Fix bug in implementation of `semantic_domain_stats_sample_rate`.

WORKSPACE

@@ -64,6 +64,63 @@ load("@rules_proto//proto:repositories.bzl", "rules_proto_dependencies", "rules_
 rules_proto_dependencies()
 rules_proto_toolchains()
 
+# TODO(b/239095455): Change to using a tfx-bsl workspace macro to load these
+# dependencies.
+# Needed by zetasql.
+PROTOBUF_COMMIT = "fde7cf7358ec7cd69e8db9be4f1fa6a5c431386a" # 3.13.0
+http_archive(
+    name = "com_google_protobuf",
+    sha256 = "e589e39ef46fb2b3b476b3ca355bd324e5984cbdfac19f0e1625f0042e99c276",
+    strip_prefix = "protobuf-%s" % PROTOBUF_COMMIT,
+    urls = [
+        "https://storage.googleapis.com/grpc-bazel-mirror/github.com/google/protobuf/archive/%s.tar.gz" % PROTOBUF_COMMIT,
+        "https://github.com/google/protobuf/archive/%s.tar.gz" % PROTOBUF_COMMIT,
+    ],
+)
+
+# Needed by abseil-py by zetasql.
+http_archive(
+    name = "six_archive",
+    urls = [
+        "http://mirror.bazel.build/pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+        "https://pypi.python.org/packages/source/s/six/six-1.10.0.tar.gz",
+    ],
+    sha256 = "105f8d68616f8248e24bf0e9372ef04d3cc10104f1980f54d57b2ce73a5ad56a",
+    strip_prefix = "six-1.10.0",
+    build_file = "//third_party:six.BUILD"
+)
+
+ABSL_COMMIT = "e1d388e7e74803050423d035e4374131b9b57919"  # lts_20210324.1
+http_archive(
+    name = "com_google_absl",
+    urls = ["https://github.com/abseil/abseil-cpp/archive/%s.zip" % ABSL_COMMIT],
+    sha256 = "baebd1536bec56ae7d7c060c20c01af89ecba2c0b1bc8992b652520655395f94",
+    strip_prefix = "abseil-cpp-%s" % ABSL_COMMIT,
+)
+
+ZETASQL_COMMIT = "5ccb05880e72ab9ff75dd6b05d7b0acce53f1ea2"  # 04/22/2021
+http_archive(
+    name = "com_google_zetasql",
+    urls = ["https://github.com/google/zetasql/archive/%s.zip" % ZETASQL_COMMIT],
+    strip_prefix = "zetasql-%s" % ZETASQL_COMMIT,
+    sha256 = "4ca4e45f457926484822701ec15ca4d0172b01d7ce43c0b34c6f3ab98c95b241",
+)
+
+load("@com_google_zetasql//bazel:zetasql_deps_step_1.bzl", "zetasql_deps_step_1")
+
+zetasql_deps_step_1()
+
+load("@com_google_zetasql//bazel:zetasql_deps_step_2.bzl", "zetasql_deps_step_2")
+
+zetasql_deps_step_2(
+    analyzer_deps = True,
+    evaluator_deps = True,
+    tools_deps = False,
+    java_deps = False,
+    testing_deps = False,
+)
+
+
 # Please add all new TensorFlow Data Validation dependencies in workspace.bzl.
 load("//tensorflow_data_validation:workspace.bzl", "tf_data_validation_workspace")
 

setup.py

@@ -73,7 +73,7 @@ def finalize_options(self):
           'installation instruction.')
     self._additional_build_options = []
     if platform.system() == 'Darwin':
-      self._additional_build_options = ['--macos_minimum_os=10.9']
+      self._additional_build_options = ['--macos_minimum_os=10.14']
     elif platform.system() == 'Windows':
       self._additional_build_options = ['--copt=-DWIN32_LEAN_AND_MEAN']
 

tensorflow_data_validation/BUILD

@@ -1,5 +1,7 @@
 licenses(["notice"])  # Apache 2.0
 
+load("@bazel_skylib//lib:selects.bzl", "selects")
+
 config_setting(
     name = "windows",
     constraint_values = [
@@ -13,12 +15,14 @@ sh_binary(
     data = select({
         ":windows": [
             "//tensorflow_data_validation/skew/protos:feature_skew_results_pb2.py",
+            "//tensorflow_data_validation/anomalies/proto:custom_validation_config_pb2.py",
             "//tensorflow_data_validation/anomalies/proto:validation_config_pb2.py",
             "//tensorflow_data_validation/anomalies/proto:validation_metadata_pb2.py",
             "//tensorflow_data_validation/pywrap:tensorflow_data_validation_extension.pyd",
         ],
         "//conditions:default": [
             "//tensorflow_data_validation/skew/protos:feature_skew_results_pb2.py",
+            "//tensorflow_data_validation/anomalies/proto:custom_validation_config_pb2.py",
             "//tensorflow_data_validation/anomalies/proto:validation_config_pb2.py",
             "//tensorflow_data_validation/anomalies/proto:validation_metadata_pb2.py",
             "//tensorflow_data_validation/pywrap:tensorflow_data_validation_extension.so",

tensorflow_data_validation/anomalies/BUILD

@@ -439,10 +439,11 @@ cc_library(
     deps = [
         ":schema",
         ":statistics_view",
-        "//tensorflow_data_validation/anomalies/proto:custom_validation_config_cc_proto",
-        "//third_party/py/tfx_bsl/cc/statistics:sql_util",
+        "//tensorflow_data_validation/anomalies/proto:custom_validation_config_proto",
         "@com_github_tensorflow_metadata//tensorflow_metadata/proto/v0:metadata_v0_proto_cc_pb2",
+        "@com_github_tfx_bsl//tfx_bsl/cc/statistics:sql_util",
         "@com_google_absl//absl/container:flat_hash_map",
+        "@com_google_absl//absl/types:optional",
         "@org_tensorflow//tensorflow/core:lib",
     ],
 )

tensorflow_data_validation/anomalies/custom_validation.cc

@@ -12,16 +12,13 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
 ==============================================================================*/
-
 #include "tensorflow_data_validation/anomalies/custom_validation.h"
 
 #include "absl/container/flat_hash_map.h"
 #include "tensorflow_data_validation/anomalies/schema_util.h"
 #include "tensorflow_data_validation/anomalies/statistics_view.h"
-#include "third_party/py/tfx_bsl/cc/statistics/sql_util.h"
+#include "tfx_bsl/cc/statistics/sql_util.h"
 #include "tensorflow/core/lib/core/errors.h"
-#include "tensorflow/core/platform/errors.h"
-#include "tensorflow/tsl/platform/errors.h"
 #include "tensorflow_metadata/proto/v0/anomalies.pb.h"
 #include "tensorflow_metadata/proto/v0/path.pb.h"
 #include "tensorflow_metadata/proto/v0/statistics.pb.h"
@@ -255,5 +252,48 @@ Status CustomValidateStatistics(
   return tensorflow::Status();
 }
 
+Status CustomValidateStatisticsWithSerializedInputs(
+    const std::string& serialized_test_statistics,
+    const std::string& serialized_base_statistics,
+    const std::string& serialized_validations,
+    const std::string& serialized_environment,
+    std::string* serialized_anomalies_proto) {
+  metadata::v0::DatasetFeatureStatisticsList test_statistics;
+  metadata::v0::DatasetFeatureStatisticsList base_statistics;
+  metadata::v0::DatasetFeatureStatisticsList* base_statistics_ptr = nullptr;
+  if (!test_statistics.ParseFromString(serialized_test_statistics)) {
+    return tensorflow::errors::InvalidArgument(
+        "Failed to parse DatasetFeatureStatistics proto.");
+  }
+  if (!serialized_base_statistics.empty()) {
+    if (!base_statistics.ParseFromString(serialized_base_statistics)) {
+      return tensorflow::errors::InvalidArgument(
+          "Failed to parse DatasetFeatureStatistics proto.");
+    }
+    base_statistics_ptr = &base_statistics;
+  }
+  CustomValidationConfig validations;
+  if (!validations.ParseFromString(serialized_validations)) {
+    return tensorflow::errors::InvalidArgument(
+        "Failed to parse CustomValidationConfig proto.");
+  }
+  absl::optional<std::string> environment = absl::nullopt;
+  if (!serialized_environment.empty()) {
+    environment = serialized_environment;
+  }
+  metadata::v0::Anomalies anomalies;
+  const tensorflow::Status status =
+      CustomValidateStatistics(test_statistics, base_statistics_ptr,
+                               validations, environment, &anomalies);
+  if (!status.ok()) {
+    return tensorflow::errors::Internal("Failed to run custom validations.");
+  }
+  if (!anomalies.SerializeToString(serialized_anomalies_proto)) {
+    return tensorflow::errors::Internal(
+        "Failed to serialize Anomalies output proto to string.");
+  }
+  return tensorflow::Status();
+}
+
 }  // namespace data_validation
 }  // namespace tensorflow

tensorflow_data_validation/anomalies/custom_validation.h

@@ -15,6 +15,7 @@ limitations under the License.
 #ifndef THIRD_PARTY_PY_TENSORFLOW_DATA_VALIDATION_ANOMALIES_CUSTOM_VALIDATION_H_
 #define THIRD_PARTY_PY_TENSORFLOW_DATA_VALIDATION_ANOMALIES_CUSTOM_VALIDATION_H_
 
+#include "absl/types/optional.h"
 #include "tensorflow_data_validation/anomalies/proto/custom_validation_config.pb.h"
 #include "tensorflow/core/lib/core/status.h"
 #include "tensorflow_metadata/proto/v0/anomalies.pb.h"
@@ -33,6 +34,15 @@ Status CustomValidateStatistics(
     const CustomValidationConfig& validations,
     const absl::optional<string> environment, metadata::v0::Anomalies* result);
 
+// Like CustomValidateStatistics but with serialized inputs. Used for doing
+// custom validation in Python.
+Status CustomValidateStatisticsWithSerializedInputs(
+    const std::string& serialized_test_statistics,
+    const std::string& serialized_base_statistics,
+    const std::string& serialized_validations,
+    const std::string& serialized_environment,
+    std::string* serialized_anomalies_proto);
+
 }  // namespace data_validation
 }  // namespace tensorflow
 

tensorflow_data_validation/api/validation_api.py

@@ -25,6 +25,7 @@
 import tensorflow as tf
 from tensorflow_data_validation import constants
 from tensorflow_data_validation import types
+from tensorflow_data_validation.anomalies.proto import custom_validation_config_pb2
 from tensorflow_data_validation.anomalies.proto import validation_config_pb2
 from tensorflow_data_validation.anomalies.proto import validation_metadata_pb2
 from tensorflow_data_validation.api import validation_options as vo
@@ -451,6 +452,58 @@ def validate_statistics_internal(
   return result
 
 
+# TODO(b/239095455): Also integrate with validate_statistics.
+def custom_validate_statistics(
+    statistics: statistics_pb2.DatasetFeatureStatisticsList,
+    validations: custom_validation_config_pb2.CustomValidationConfig,
+    baseline_statistics: Optional[
+        statistics_pb2.DatasetFeatureStatisticsList] = None,
+    environment: Optional[str] = None) -> anomalies_pb2.Anomalies:
+  """Validates the input statistics with the user-supplied SQL queries.
+
+  If the SQL query from a user-supplied validation returns False, TFDV will
+  return an anomaly for that validation. In single feature valdiations, the test
+  feature will be mapped to `feature` in the SQL query. In two feature
+  validations, the test feature will be mapped to `feature_test` in the SQL
+  query, and the base feature will be mapped to `feature_base`.
+
+  If an optional `environment` is supplied, TFDV will run validations with
+  that environment specified and validations with no environment specified.
+
+  NOTE: This function is not supported on Windows.
+
+  Args:
+    statistics: A DatasetFeatureStatisticsList protocol buffer that holds the
+      statistics to validate.
+    validations: Configuration that specifies the dataset(s) and feature(s) to
+      validate and the SQL query to use for the validation. The SQL query must
+      return a boolean value.
+    baseline_statistics: An optional DatasetFeatureStatisticsList protocol
+      buffer that holds the baseline statistics used when validating feature
+      pairs.
+    environment: If supplied, TFDV will run validations with that
+      environment specified and validations with no environment specified. If
+      not supplied, TFDV will run all validations.
+  Returns:
+    An Anomalies protocol buffer.
+  """
+  serialized_statistics = statistics.SerializeToString()
+  serialized_baseline_statistics = (
+      baseline_statistics.SerializeToString()
+      if baseline_statistics is not None else '')
+  serialized_validations = validations.SerializeToString()
+  environment = '' if environment is None else environment
+  serialized_anomalies = (
+      pywrap_tensorflow_data_validation.CustomValidateStatistics(
+          tf.compat.as_bytes(serialized_statistics),
+          tf.compat.as_bytes(serialized_baseline_statistics),
+          tf.compat.as_bytes(serialized_validations),
+          tf.compat.as_bytes(environment)))
+  result = anomalies_pb2.Anomalies()
+  result.ParseFromString(serialized_anomalies)
+  return result
+
+
 def _remove_features_missing_common_stats(
     stats: statistics_pb2.DatasetFeatureStatistics
 ) -> statistics_pb2.DatasetFeatureStatistics: