From 65bd0fb3a2bb4e958fe06546211d07b33da5e7aa Mon Sep 17 00:00:00 2001 From: Sara Robinson Date: Thu, 23 Oct 2025 13:32:54 -0700 Subject: [PATCH] chore: GenAI SDK client - introduce subnamespaces for types PiperOrigin-RevId: 823168033 --- tests/unit/vertexai/genai/replays/conftest.py | 4 +- .../genai/replays/test_evaluate_instances.py | 30 +- tests/unit/vertexai/genai/test_evals.py | 61 +- vertexai/_genai/_bigquery_utils.py | 49 + vertexai/_genai/_evals_common.py | 41 +- vertexai/_genai/_evals_data_converters.py | 2 +- vertexai/_genai/_evals_metric_handlers.py | 30 +- vertexai/_genai/_evals_metric_loaders.py | 413 ++++ vertexai/_genai/_evals_utils.py | 178 +- vertexai/_genai/_evals_visualization.py | 3 +- vertexai/_genai/_gcs_utils.py | 192 ++ vertexai/_genai/_transformers.py | 2 +- vertexai/_genai/evals.py | 4 +- vertexai/_genai/types/__init__.py | 1855 +++++++++++++++++ vertexai/_genai/types/agent_engines.py | 16 + vertexai/_genai/{types.py => types/common.py} | 223 +- vertexai/_genai/types/evals.py | 202 ++ 17 files changed, 2849 insertions(+), 456 deletions(-) create mode 100644 vertexai/_genai/_bigquery_utils.py create mode 100644 vertexai/_genai/_evals_metric_loaders.py create mode 100644 vertexai/_genai/_gcs_utils.py create mode 100644 vertexai/_genai/types/__init__.py create mode 100644 vertexai/_genai/types/agent_engines.py rename vertexai/_genai/{types.py => types/common.py} (98%) create mode 100644 vertexai/_genai/types/evals.py diff --git a/tests/unit/vertexai/genai/replays/conftest.py b/tests/unit/vertexai/genai/replays/conftest.py index 733d175a9c..eafc155e13 100644 --- a/tests/unit/vertexai/genai/replays/conftest.py +++ b/tests/unit/vertexai/genai/replays/conftest.py @@ -25,7 +25,7 @@ from google.cloud import storage, bigquery from google.genai import _replay_api_client from google.genai import client as google_genai_client_module -from vertexai._genai import _evals_utils +from vertexai._genai import _gcs_utils from vertexai._genai import prompt_optimizer import pytest @@ -246,7 +246,7 @@ def client(use_vertex, replays_prefix, http_options, request): mock_bigquery_client.return_value = mock.MagicMock() with mock.patch.object( - _evals_utils.GcsUtils, "read_file_contents" + _gcs_utils.GcsUtils, "read_file_contents" ) as mock_read_file_contents: mock_read_file_contents.side_effect = ( _mock_read_file_contents_side_effect diff --git a/tests/unit/vertexai/genai/replays/test_evaluate_instances.py b/tests/unit/vertexai/genai/replays/test_evaluate_instances.py index 33128fffb2..76c04d16af 100644 --- a/tests/unit/vertexai/genai/replays/test_evaluate_instances.py +++ b/tests/unit/vertexai/genai/replays/test_evaluate_instances.py @@ -103,9 +103,9 @@ def test_pointwise_metric_with_agent_data(client): """Tests the _evaluate_instances method with PointwiseMetricInput and agent_data.""" instance_dict = {"prompt": "What is the capital of France?", "response": "Paris"} json_instance = json.dumps(instance_dict) - agent_data = types.AgentData( - agent_config=types.AgentConfig( - tools=types.Tools( + agent_data = types.evals.AgentData( + agent_config=types.evals.AgentConfig( + tools=types.evals.Tools( tool=[ genai_types.Tool( function_declarations=[ @@ -114,15 +114,15 @@ def test_pointwise_metric_with_agent_data(client): ) ] ), - developer_instruction=types.InstanceData(text="instruction"), + developer_instruction=types.evals.InstanceData(text="instruction"), ), - events=types.Events( + events=types.evals.Events( event=[genai_types.Content(parts=[genai_types.Part(text="hello")])] ), ) instance = types.EvaluationInstance( - prompt=types.InstanceData(text="What is the capital of France?"), - response=types.InstanceData(text="Paris"), + prompt=types.evals.InstanceData(text="What is the capital of France?"), + response=types.evals.InstanceData(text="Paris"), agent_data=agent_data, ) @@ -144,9 +144,9 @@ def test_pointwise_metric_with_agent_data(client): def test_predefined_metric_with_agent_data(client): """Tests the _evaluate_instances method with predefined metric and agent_data.""" - agent_data = types.AgentData( - agent_config=types.AgentConfig( - tools=types.Tools( + agent_data = types.evals.AgentData( + agent_config=types.evals.AgentConfig( + tools=types.evals.Tools( tool=[ genai_types.Tool( function_declarations=[ @@ -155,16 +155,16 @@ def test_predefined_metric_with_agent_data(client): ) ] ), - developer_instruction=types.InstanceData(text="instruction"), + developer_instruction=types.evals.InstanceData(text="instruction"), ), - events=types.Events( + events=types.evals.Events( event=[genai_types.Content(parts=[genai_types.Part(text="hello")])] ), ) instance = types.EvaluationInstance( - prompt=types.InstanceData(text="What is the capital of France?"), - response=types.InstanceData(text="Paris"), - reference=types.InstanceData(text="Paris"), + prompt=types.evals.InstanceData(text="What is the capital of France?"), + response=types.evals.InstanceData(text="Paris"), + reference=types.evals.InstanceData(text="Paris"), agent_data=agent_data, ) diff --git a/tests/unit/vertexai/genai/test_evals.py b/tests/unit/vertexai/genai/test_evals.py index 7f461566ab..1453a555ae 100644 --- a/tests/unit/vertexai/genai/test_evals.py +++ b/tests/unit/vertexai/genai/test_evals.py @@ -27,6 +27,8 @@ from vertexai._genai import _evals_data_converters from vertexai._genai import _evals_metric_handlers from vertexai._genai import _evals_visualization +from vertexai._genai import _evals_metric_loaders +from vertexai._genai import _gcs_utils from vertexai._genai import _observability_data_converter from vertexai._genai import evals from vertexai._genai import types as vertexai_genai_types @@ -76,9 +78,9 @@ def mock_eval_dependencies(mock_api_client_fixture): ) as mock_bq_client, mock.patch( "vertexai._genai.evals.Evals.evaluate_instances" ) as mock_evaluate_instances, mock.patch( - "vertexai._genai._evals_utils.GcsUtils.upload_json_to_prefix" + "vertexai._genai._gcs_utils.GcsUtils.upload_json_to_prefix" ) as mock_upload_to_gcs, mock.patch( - "vertexai._genai._evals_utils.LazyLoadedPrebuiltMetric._fetch_and_parse" + "vertexai._genai._evals_metric_loaders.LazyLoadedPrebuiltMetric._fetch_and_parse" ) as mock_fetch_prebuilt_metric: def mock_evaluate_instances_side_effect(*args, **kwargs): @@ -235,7 +237,7 @@ def test_display_evaluation_result_with_agent_trace_prefixes(self, mock_is_ipyth ) eval_result = vertexai_genai_types.EvaluationResult( evaluation_dataset=[eval_dataset], - agent_info=vertexai_genai_types.AgentInfo(name="test_agent"), + agent_info=vertexai_genai_types.evals.AgentInfo(name="test_agent"), eval_case_results=[ vertexai_genai_types.EvalCaseResult( eval_case_index=0, @@ -284,7 +286,7 @@ def setup_method(self): self.client = vertexai.Client(project=_TEST_PROJECT, location=_TEST_LOCATION) @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_string_model_success( self, mock_eval_dataset_loader, mock_models ): @@ -327,7 +329,7 @@ def test_inference_with_string_model_success( assert inference_result.candidate_name == "gemini-pro" assert inference_result.gcs_source is None - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_callable_model_sets_candidate_name( self, mock_eval_dataset_loader ): @@ -346,7 +348,7 @@ def my_model_fn(contents): assert inference_result.candidate_name == "my_model_fn" assert inference_result.gcs_source is None - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_lambda_model_candidate_name_is_none( self, mock_eval_dataset_loader ): @@ -368,7 +370,7 @@ def test_inference_with_lambda_model_candidate_name_is_none( ) assert inference_result.gcs_source is None - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_callable_model_success(self, mock_eval_dataset_loader): mock_df = pd.DataFrame({"prompt": ["test prompt"]}) mock_eval_dataset_loader.return_value.load.return_value = mock_df.to_dict( @@ -396,7 +398,7 @@ def mock_model_fn(contents): assert inference_result.gcs_source is None @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_prompt_template( self, mock_eval_dataset_loader, mock_models ): @@ -443,8 +445,8 @@ def test_inference_with_prompt_template( assert inference_result.gcs_source is None @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") - @mock.patch.object(_evals_utils, "GcsUtils") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") + @mock.patch.object(_gcs_utils, "GcsUtils") def test_inference_with_gcs_destination( self, mock_gcs_utils, mock_eval_dataset_loader, mock_models ): @@ -497,7 +499,7 @@ def test_inference_with_gcs_destination( ) @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") @mock.patch("pandas.DataFrame.to_json") @mock.patch("os.makedirs") def test_inference_with_local_destination( @@ -549,7 +551,7 @@ def test_inference_with_local_destination( assert inference_result.gcs_source is None @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_from_request_column_save_to_local_dir( self, mock_eval_dataset_loader, mock_models ): @@ -783,7 +785,7 @@ def test_inference_from_local_csv_file(self, mock_models): assert inference_result.gcs_source is None @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_row_level_config_overrides( self, mock_eval_dataset_loader, mock_models ): @@ -968,7 +970,7 @@ def mock_generate_content_logic(*args, **kwargs): assert inference_result.gcs_source is None @mock.patch.object(_evals_common, "Models") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_inference_with_multimodal_content( self, mock_eval_dataset_loader, mock_models ): @@ -1044,7 +1046,7 @@ def test_inference_with_multimodal_content( assert inference_result.candidate_name == "gemini-pro" assert inference_result.gcs_source is None - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") @mock.patch("vertexai._genai._evals_common.vertexai.Client") def test_run_inference_with_agent_engine_and_session_inputs_dict( self, @@ -1140,7 +1142,7 @@ async def _async_iterator(iterable): assert inference_result.candidate_name == "agent" assert inference_result.gcs_source is None - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") @mock.patch("vertexai._genai._evals_common.vertexai.Client") def test_run_inference_with_agent_engine_and_session_inputs_literal_string( self, @@ -1423,7 +1425,7 @@ def test_run_inference_with_litellm_import_error(self, mock_api_client_fixture): @mock.patch.object(_evals_common, "_is_gemini_model") @mock.patch.object(_evals_common, "_is_litellm_model") @mock.patch.object(_evals_common, "_is_litellm_vertex_maas_model") - @mock.patch.object(_evals_utils, "EvalDatasetLoader") + @mock.patch.object(_evals_metric_loaders, "EvalDatasetLoader") def test_run_inference_with_litellm_parsing( self, mock_eval_dataset_loader, @@ -2783,7 +2785,7 @@ def test_agent_info_creation(self): ) ] ) - agent_info = vertexai_genai_types.AgentInfo( + agent_info = vertexai_genai_types.evals.AgentInfo( name="agent1", instruction="instruction1", description="description1", @@ -2827,7 +2829,7 @@ def test_eval_case_with_agent_eval_fields(self): ) ] ) - agent_info = vertexai_genai_types.AgentInfo( + agent_info = vertexai_genai_types.evals.AgentInfo( name="agent1", instruction="instruction1", tool_declarations=[tool], @@ -2933,7 +2935,7 @@ def test_metric_name_validation_lowercase(self): metric = vertexai_genai_types.Metric(name="UPPERCASEMetric") assert metric.name == "uppercasemetric" - @mock.patch("vertexai._genai.types.yaml.dump") + @mock.patch("vertexai._genai.types.common.yaml.dump") @mock.patch("builtins.open", new_callable=mock.mock_open) def test_metric_to_yaml_file_with_version_and_set_fields( self, mock_open_file, mock_yaml_dump @@ -2970,7 +2972,7 @@ def test_metric_to_yaml_file_with_version_and_set_fields( allow_unicode=True, ) - @mock.patch("vertexai._genai.types.yaml.dump") + @mock.patch("vertexai._genai.types.common.yaml.dump") @mock.patch("builtins.open", new_callable=mock.mock_open) def test_metric_to_yaml_file_without_version_minimal_fields( self, mock_open_file, mock_yaml_dump @@ -2991,7 +2993,7 @@ def test_metric_to_yaml_file_without_version_minimal_fields( allow_unicode=True, ) - @mock.patch("vertexai._genai.types.yaml", None) + @mock.patch("vertexai._genai.types.common.yaml", None) def test_metric_to_yaml_file_raises_importerror_if_yaml_is_none(self): metric_obj = vertexai_genai_types.Metric(name="ErrorMetric") with pytest.raises( @@ -3699,7 +3701,7 @@ def test_eval_case_to_agent_data(self): ) ] ) - agent_info = vertexai_genai_types.AgentInfo( + agent_info = vertexai_genai_types.evals.AgentInfo( name="agent1", instruction="instruction1", tool_declarations=[tool], @@ -3797,7 +3799,6 @@ def setup_method(self): importlib.reload(aiplatform_initializer) importlib.reload(aiplatform) importlib.reload(vertexai) - importlib.reload(genai_types) importlib.reload(vertexai_genai_types) importlib.reload(_evals_data_converters) importlib.reload(_evals_metric_handlers) @@ -4346,7 +4347,9 @@ def test_execute_evaluation_with_openai_schema( name="test_metric", prompt_template="Evaluate: {response}" ) - with mock.patch.object(_evals_utils, "EvalDatasetLoader") as mock_loader_class: + with mock.patch.object( + _evals_metric_loaders, "EvalDatasetLoader" + ) as mock_loader_class: mock_loader_instance = mock_loader_class.return_value mock_loader_instance.load.return_value = mock_openai_raw_data @@ -4599,7 +4602,7 @@ def test_execute_evaluation_lazy_loaded_prebuilt_metric_instance( eval_dataset_df=dataset_df ) - lazy_metric_instance = _evals_utils.LazyLoadedPrebuiltMetric( + lazy_metric_instance = _evals_metric_loaders.LazyLoadedPrebuiltMetric( name="fluency", version="v1" ) @@ -4799,7 +4802,7 @@ def test_execute_evaluation_adds_creation_timestamp( class TestEvaluationDataset: """Contains set of tests for the EvaluationDataset class methods.""" - @mock.patch.object(_evals_utils, "GcsUtils") + @mock.patch.object(_gcs_utils, "GcsUtils") def test_load_from_observability_eval_cases(self, mock_gcs_utils): """Tests that load_from_observability_eval_cases reads data from GCS.""" @@ -4851,7 +4854,7 @@ def read_file_contents_side_effect(src: str) -> str: ), ) - @mock.patch.object(_evals_utils, "GcsUtils") + @mock.patch.object(_gcs_utils, "GcsUtils") def test_load_from_observability_eval_cases_no_system_instruction( self, mock_gcs_utils ): @@ -4903,7 +4906,7 @@ def read_file_contents_side_effect(src: str) -> str: ), ) - @mock.patch.object(_evals_utils, "GcsUtils") + @mock.patch.object(_gcs_utils, "GcsUtils") def test_load_from_observability_eval_cases_multiple_cases(self, mock_gcs_utils): """Test load_from_observability_eval_cases can handle multiple cases.""" diff --git a/vertexai/_genai/_bigquery_utils.py b/vertexai/_genai/_bigquery_utils.py new file mode 100644 index 0000000000..94813df432 --- /dev/null +++ b/vertexai/_genai/_bigquery_utils.py @@ -0,0 +1,49 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import logging + +from google.cloud import bigquery +from google.genai._api_client import BaseApiClient +import pandas as pd + + +logger = logging.getLogger(__name__) + + +class BigQueryUtils: + """Handles BigQuery operations.""" + + def __init__(self, api_client: BaseApiClient): + self.api_client = api_client + self.bigquery_client = bigquery.Client( + project=self.api_client.project, + credentials=self.api_client._credentials, + ) + + def load_bigquery_to_dataframe(self, table_uri: str) -> "pd.DataFrame": + """Loads data from a BigQuery table into a DataFrame.""" + table = self.bigquery_client.get_table(table_uri) + return self.bigquery_client.list_rows(table).to_dataframe() + + def upload_dataframe_to_bigquery( + self, df: "pd.DataFrame", bq_table_uri: str + ) -> None: + """Uploads a Pandas DataFrame to a BigQuery table.""" + job = self.bigquery_client.load_table_from_dataframe(df, bq_table_uri) + job.result() + logger.info( + f"DataFrame successfully uploaded to BigQuery table: {bq_table_uri}" + ) diff --git a/vertexai/_genai/_evals_common.py b/vertexai/_genai/_evals_common.py index 716ab1be04..89357d9a1d 100644 --- a/vertexai/_genai/_evals_common.py +++ b/vertexai/_genai/_evals_common.py @@ -35,7 +35,8 @@ from . import _evals_constant from . import _evals_data_converters from . import _evals_metric_handlers -from . import _evals_utils +from . import _evals_metric_loaders +from . import _gcs_utils from . import evals from . import types @@ -693,7 +694,7 @@ def _load_dataframe( """Loads and prepares the prompt dataset for inference.""" logger.info("Loading prompt dataset from: %s", src) try: - loader = _evals_utils.EvalDatasetLoader(api_client=api_client) + loader = _evals_metric_loaders.EvalDatasetLoader(api_client=api_client) dataset_list_of_dicts = loader.load(src) if not dataset_list_of_dicts: raise ValueError("Prompt dataset 'prompt_dataset' must not be empty.") @@ -805,7 +806,7 @@ def _execute_inference( if dest: file_name = "inference_results.jsonl" if model else "agent_run_results.jsonl" - is_gcs_path = dest.startswith(_evals_utils.GCS_PREFIX) + is_gcs_path = dest.startswith(_gcs_utils.GCS_PREFIX) if is_gcs_path: full_dest_path = os.path.join(dest, file_name) @@ -816,7 +817,7 @@ def _execute_inference( logger.info("Saving inference / agent run results to: %s", full_dest_path) try: if is_gcs_path: - _evals_utils.GcsUtils(api_client=api_client).upload_dataframe( + _gcs_utils.GcsUtils(api_client=api_client).upload_dataframe( df=results_df, gcs_destination_blob_path=full_dest_path, file_type="jsonl", @@ -858,8 +859,8 @@ def _get_dataset_source( def _resolve_dataset_inputs( dataset: list[types.EvaluationDataset], dataset_schema: Optional[Literal["GEMINI", "FLATTEN", "OPENAI"]], - loader: "_evals_utils.EvalDatasetLoader", - agent_info: Optional[types.AgentInfo] = None, + loader: "_evals_metric_loaders.EvalDatasetLoader", + agent_info: Optional[types.evals.AgentInfo] = None, ) -> tuple[types.EvaluationDataset, int]: """Loads and processes single or multiple datasets for evaluation. @@ -943,7 +944,9 @@ def _resolve_evaluation_run_metrics( for metric_instance in metrics: if isinstance(metric_instance, types.EvaluationRunMetric): resolved_metrics_list.append(metric_instance) - elif isinstance(metric_instance, _evals_utils.LazyLoadedPrebuiltMetric): + elif isinstance( + metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric + ): try: resolved_metric = metric_instance.resolve(api_client=api_client) if resolved_metric.name: @@ -969,10 +972,10 @@ def _resolve_evaluation_run_metrics( try: metric_name_str = str(metric_instance) lazy_metric_instance = getattr( - _evals_utils.RubricMetric, metric_name_str.upper() + _evals_metric_loaders.RubricMetric, metric_name_str.upper() ) if isinstance( - lazy_metric_instance, _evals_utils.LazyLoadedPrebuiltMetric + lazy_metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric ): resolved_metric = lazy_metric_instance.resolve( api_client=api_client @@ -1006,7 +1009,7 @@ def _resolve_metrics( """Resolves a list of metric instances, loading RubricMetric if necessary.""" resolved_metrics_list = [] for metric_instance in metrics: - if isinstance(metric_instance, _evals_utils.LazyLoadedPrebuiltMetric): + if isinstance(metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric): try: resolved_metrics_list.append( metric_instance.resolve(api_client=api_client) @@ -1025,10 +1028,10 @@ def _resolve_metrics( try: metric_name_str = str(metric_instance) lazy_metric_instance = getattr( - _evals_utils.RubricMetric, metric_name_str.upper() + _evals_metric_loaders.RubricMetric, metric_name_str.upper() ) if isinstance( - lazy_metric_instance, _evals_utils.LazyLoadedPrebuiltMetric + lazy_metric_instance, _evals_metric_loaders.LazyLoadedPrebuiltMetric ): resolved_metrics_list.append( lazy_metric_instance.resolve(api_client=api_client) @@ -1100,14 +1103,14 @@ def _execute_evaluation( else: deduped_candidate_names.append(name) - loader = _evals_utils.EvalDatasetLoader(api_client=api_client) + loader = _evals_metric_loaders.EvalDatasetLoader(api_client=api_client) agent_info = kwargs.get("agent_info", None) validated_agent_info = None if agent_info: if isinstance(agent_info, dict): - validated_agent_info = types.AgentInfo.model_validate(agent_info) - elif isinstance(agent_info, types.AgentInfo): + validated_agent_info = types.evals.AgentInfo.model_validate(agent_info) + elif isinstance(agent_info, types.evals.AgentInfo): validated_agent_info = agent_info else: raise TypeError( @@ -1154,7 +1157,7 @@ def _execute_evaluation( logger.info("Evaluation run completed.") if dest: - uploaded_path = _evals_utils.GcsUtils( + uploaded_path = _gcs_utils.GcsUtils( api_client=api_client ).upload_json_to_prefix( data=evaluation_result.model_dump( @@ -1334,7 +1337,7 @@ def _convert_gcs_to_evaluation_item_result( ) -> types.EvaluationItemResult: """Converts a json file to an EvaluationItemResult.""" logger.info("Loading evaluation item result from GCS: %s", gcs_uri) - gcs_utils = _evals_utils.GcsUtils(api_client=api_client) + gcs_utils = _gcs_utils.GcsUtils(api_client=api_client) try: eval_item_data = json.loads(gcs_utils.read_file_contents(gcs_uri)) return types.EvaluationItemResult(**eval_item_data) @@ -1351,7 +1354,7 @@ def _convert_gcs_to_evaluation_item_request( ) -> types.EvaluationItemRequest: """Converts a json file to an EvaluationItemRequest.""" logger.info("Loading evaluation item request from GCS: %s", gcs_uri) - gcs_utils = _evals_utils.GcsUtils(api_client=api_client) + gcs_utils = _gcs_utils.GcsUtils(api_client=api_client) try: eval_item_data = json.loads(gcs_utils.read_file_contents(gcs_uri)) return types.EvaluationItemRequest(**eval_item_data) @@ -1634,7 +1637,7 @@ def _create_evaluation_set_from_dataframe( ) ) logger.info("Writing evaluation item requests to GCS.") - gcs_utils = _evals_utils.GcsUtils(api_client=api_client) + gcs_utils = _gcs_utils.GcsUtils(api_client=api_client) evals_module = evals.Evals(api_client_=api_client) eval_items = [] for eval_item_request in eval_item_requests: diff --git a/vertexai/_genai/_evals_data_converters.py b/vertexai/_genai/_evals_data_converters.py index e7135317bc..340bf72965 100644 --- a/vertexai/_genai/_evals_data_converters.py +++ b/vertexai/_genai/_evals_data_converters.py @@ -689,7 +689,7 @@ def _validate_case_consistency( def merge_response_datasets_into_canonical_format( raw_datasets: list[list[dict[str, Any]]], schemas: list[str], - agent_info: Optional[types.AgentInfo] = None, + agent_info: Optional[types.evals.AgentInfo] = None, ) -> types.EvaluationDataset: """Merges multiple raw response datasets into a single EvaluationDataset. diff --git a/vertexai/_genai/_evals_metric_handlers.py b/vertexai/_genai/_evals_metric_handlers.py index d73fafc3f9..acbeda2afd 100644 --- a/vertexai/_genai/_evals_metric_handlers.py +++ b/vertexai/_genai/_evals_metric_handlers.py @@ -837,18 +837,18 @@ def __init__(self, module: "evals.Evals", metric: types.Metric): @staticmethod def _content_to_instance_data( content: Optional[genai_types.Content], - ) -> Optional[types.InstanceData]: + ) -> Optional[types.evals.InstanceData]: """Converts a genai_types.Content object to a types.InstanceData object.""" if not content: return None - return types.InstanceData( - contents=types.InstanceDataContents(contents=[content]) + return types.evals.InstanceData( + contents=types.evals.InstanceDataContents(contents=[content]) ) @staticmethod def _eval_case_to_agent_data( eval_case: types.EvalCase, - ) -> Optional[types.AgentData]: + ) -> Optional[types.evals.AgentData]: """Converts an EvalCase object to an AgentData object.""" if not eval_case.agent_info and not eval_case.intermediate_events: return None @@ -860,12 +860,14 @@ def _eval_case_to_agent_data( if eval_case.agent_info: agent_info = eval_case.agent_info if agent_info.instruction: - developer_instruction = types.InstanceData(text=agent_info.instruction) + developer_instruction = types.evals.InstanceData( + text=agent_info.instruction + ) if agent_info.tool_declarations: tool_declarations = agent_info.tool_declarations - tools = types.Tools(tool=tool_declarations) + tools = types.evals.Tools(tool=tool_declarations) if tools or developer_instruction: - agent_config = types.AgentConfig( + agent_config = types.evals.AgentConfig( tools=tools, developer_instruction=developer_instruction, ) @@ -877,15 +879,15 @@ def _eval_case_to_agent_data( if event.content ] if event_contents: - events = types.Events(event=event_contents) + events = types.evals.Events(event=event_contents) if events: - return types.AgentData( + return types.evals.AgentData( agent_config=agent_config, events=events, ) else: - return types.AgentData( + return types.evals.AgentData( agent_config=agent_config, events_text="", ) @@ -918,8 +920,8 @@ def _build_request_payload( if eval_case.prompt: prompt_contents.append(eval_case.prompt) - prompt_instance_data = types.InstanceData( - contents=types.InstanceDataContents(contents=prompt_contents) + prompt_instance_data = types.evals.InstanceData( + contents=types.evals.InstanceDataContents(contents=prompt_contents) ) else: prompt_instance_data = PredefinedMetricHandler._content_to_instance_data( @@ -929,7 +931,9 @@ def _build_request_payload( other_data_map = {} if hasattr(eval_case, "context") and eval_case.context: if isinstance(eval_case.context, str): - other_data_map["context"] = types.InstanceData(text=eval_case.context) + other_data_map["context"] = types.evals.InstanceData( + text=eval_case.context + ) elif isinstance(eval_case.context, genai_types.Content): other_data_map["context"] = ( PredefinedMetricHandler._content_to_instance_data(eval_case.context) diff --git a/vertexai/_genai/_evals_metric_loaders.py b/vertexai/_genai/_evals_metric_loaders.py new file mode 100644 index 0000000000..ecc4312317 --- /dev/null +++ b/vertexai/_genai/_evals_metric_loaders.py @@ -0,0 +1,413 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +"""Utility functions for evals.""" + +import json +import logging +import os +import re +from typing import Any, Optional, Union, TYPE_CHECKING + +from google.genai._api_client import BaseApiClient +import pandas as pd +import yaml + +from . import _evals_constant +from . import _gcs_utils +from . import _bigquery_utils + +if TYPE_CHECKING: + from . import types + + +logger = logging.getLogger(__name__) + + +GCS_PREFIX = "gs://" +BQ_PREFIX = "bq://" + + +class EvalDatasetLoader: + """A loader for datasets from various sources, using a shared client.""" + + def __init__(self, api_client: BaseApiClient): + self.api_client = api_client + self.gcs_utils = _gcs_utils.GcsUtils(self.api_client) + self.bigquery_utils = _bigquery_utils.BigQueryUtils(self.api_client) + + def _load_file( + self, filepath: str, file_type: str + ) -> Union[list[dict[str, Any]], Any]: + """Loads data from a file into a list of dictionaries.""" + if filepath.startswith(GCS_PREFIX): + df = self.gcs_utils.read_gcs_file_to_dataframe(filepath, file_type) + return df.to_dict(orient="records") + else: + if file_type == "jsonl": + df = pd.read_json(filepath, lines=True) + return df.to_dict(orient="records") + elif file_type == "csv": + df = pd.read_csv(filepath, encoding="utf-8") + return df.to_dict(orient="records") + else: + raise ValueError( + f"Unsupported file type: '{file_type}'. Please provide 'jsonl' or" + " 'csv'." + ) + + def load( + self, source: Union[str, "pd.DataFrame"] + ) -> Union[list[dict[str, Any]], Any]: + """Loads dataset from various sources into a list of dictionaries.""" + if isinstance(source, pd.DataFrame): + return source.to_dict(orient="records") + elif isinstance(source, str): + if source.startswith(BQ_PREFIX): + df = self.bigquery_utils.load_bigquery_to_dataframe( + source[len(BQ_PREFIX) :] + ) + return df.to_dict(orient="records") + + _, extension = os.path.splitext(source) + file_type = extension.lower()[1:] + + if file_type == "jsonl": + return self._load_file(source, "jsonl") + elif file_type == "csv": + return self._load_file(source, "csv") + else: + raise TypeError( + f"Unsupported file type: {file_type} from {source}. Please" + " provide a valid GCS path with `jsonl` or `csv` suffix, " + "a local file path, or a valid BigQuery table URI." + ) + else: + raise TypeError( + "Unsupported dataset type. Must be a `pd.DataFrame`, Python" + " a valid GCS path with `jsonl` or `csv` suffix, a local" + " file path, or a valid BigQuery table URI." + ) + + +class LazyLoadedPrebuiltMetric: + """A proxy object representing a prebuilt metric to be loaded on demand. + + This can resolve to either an API Predefined Metric or an LLM Metric + loaded from GCS. + """ + + _cache: dict[str, "types.Metric"] = {} + _base_gcs_path = ( + "gs://vertex-ai-generative-ai-eval-sdk-resources/metrics/{metric_name}/" + ) + + def __init__(self, name: str, version: Optional[str] = None, **kwargs): + self.name = name.upper() + self.version = version + self.metric_kwargs = kwargs + self._resolved_metric: Optional["types.Metric"] = None + + def _get_api_metric_spec_name(self) -> Optional[str]: + """Constructs the metric_spec_name for API Predefined Metrics.""" + base_name = self.name.lower() + if self.version: + # Explicit version provided + version = self.version.lower() + potential_name = f"{base_name}_{version}" + return ( + potential_name + if potential_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS + else None + ) + else: + # Default versioning: Try _v1, then base name + v1_name = f"{base_name}_v1" + if v1_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS: + return v1_name + if base_name in _evals_constant.SUPPORTED_PREDEFINED_METRICS: + return base_name + return None + + def _resolve_api_predefined(self) -> Optional["types.Metric"]: + """Attempts to resolve as an API Predefined Metric.""" + from . import types + + metric_spec_name = self._get_api_metric_spec_name() + if metric_spec_name: + logger.info( + "Resolving '%s' as API Predefined Metric with spec name: %s", + self.name, + metric_spec_name, + ) + return types.Metric(name=metric_spec_name, **self.metric_kwargs) + return None + + def _get_latest_version_uri(self, api_client: Any, metric_gcs_dir: str) -> str: + """Lists files in GCS directory and determines the latest version URI.""" + gcs_utils = _gcs_utils.GcsUtils(api_client) + bucket_name, prefix = gcs_utils.parse_gcs_path(metric_gcs_dir) + + blobs = gcs_utils.storage_client.list_blobs(bucket_name, prefix=prefix) + + version_files: list[dict[str, Union[list[int], str]]] = ( + [] + ) # {'version_parts': [1,0,0], 'filename': 'v1.0.0.yaml'} + + version_pattern = re.compile( + r"v(\d+)(?:\.(\d+))?(?:\.(\d+))?\.(yaml|yml|json)$", re.IGNORECASE + ) + + for blob in blobs: + match = version_pattern.match(os.path.basename(blob.name)) + if match: + major = int(match.group(1)) + minor = int(match.group(2)) if match.group(2) else 0 + patch = int(match.group(3)) if match.group(3) else 0 + version_files.append( + { + "version_parts": [major, minor, patch], + "filename": os.path.basename(blob.name), + } + ) + + if not version_files: + raise IOError(f"No versioned metric files found in {metric_gcs_dir}") + + version_files.sort(key=lambda x: x["version_parts"], reverse=True) + + latest_filename = version_files[0]["filename"] + return os.path.join(metric_gcs_dir, latest_filename) + + def _fetch_and_parse(self, api_client: Any) -> "types.LLMMetric": + """Fetches and parses the metric definition from GCS.""" + + from . import types + + metric_gcs_dir = self._base_gcs_path.format(metric_name=self.name.lower()) + uri: str + if self.version == "latest" or self.version is None: + uri = self._get_latest_version_uri(api_client, metric_gcs_dir) + resolved_version_match = re.match( + r"(v\d+(?:\.\d+)*)\.(?:yaml|yml|json)", + os.path.basename(uri), + re.IGNORECASE, + ) + if resolved_version_match: + self.version = resolved_version_match.group(1) + else: + # Fallback if regex fails + self.version = os.path.splitext(os.path.basename(uri))[0] + else: + yaml_uri = os.path.join(metric_gcs_dir, f"{self.version}.yaml") + json_uri = os.path.join(metric_gcs_dir, f"{self.version}.json") + + gcs_utils = _gcs_utils.GcsUtils(api_client) + try: + bucket_name, blob_path = gcs_utils.parse_gcs_path(yaml_uri) + if ( + gcs_utils.storage_client.bucket(bucket_name) + .blob(blob_path) + .exists() + ): + uri = yaml_uri + else: + bucket_name_json, blob_path_json = gcs_utils.parse_gcs_path( + json_uri + ) + if ( + gcs_utils.storage_client.bucket(bucket_name_json) + .blob(blob_path_json) + .exists() + ): + uri = json_uri + else: + raise IOError( + f"Metric file for version '{self.version}' " + f"not found as .yaml or .json in {metric_gcs_dir}" + ) + except Exception as e: + raise IOError( + f"Error checking for metric file version '{self.version}' in" + f" {metric_gcs_dir}: {e}" + ) from e + + logger.info( + "Fetching predefined metric '%s@%s' from %s...", + self.name, + self.version, + uri, + ) + + gcs_utils = _gcs_utils.GcsUtils(api_client) + content_str = gcs_utils.read_file_contents(uri) + + file_extension = os.path.splitext(uri)[1].lower() + data: dict[str, Any] + if file_extension == ".yaml" or file_extension == ".yml": + if yaml is None: + raise ImportError( + "YAML parsing requires the pyyaml library. Please install it" + " with `pip install google-cloud-aiplatform[evaluation]`." + ) + data = yaml.safe_load(content_str) + elif file_extension == ".json": + data = json.loads(content_str) + else: + raise ValueError(f"Unsupported file extension: {file_extension}") + + if not isinstance(data, dict): + raise ValueError("Metric config content did not parse into a dictionary.") + + metric_obj = types.LLMMetric.model_validate({**data, **self.metric_kwargs}) + metric_obj._is_predefined = True + metric_obj._config_source = uri + metric_obj._version = self.version + return metric_obj + + def resolve(self, api_client: Any) -> "types.Metric": + """Resolves the metric by checking API Predefined, then GCS, caching results.""" + if self._resolved_metric: + return self._resolved_metric + + cache_key = f"{self.name}@{self.version or 'default'}" + if cache_key in LazyLoadedPrebuiltMetric._cache: + self._resolved_metric = LazyLoadedPrebuiltMetric._cache[cache_key] + logger.debug("Metric '%s' found in cache.", cache_key) + return self._resolved_metric + + # Try resolving as API Predefined Metric first + api_metric = self._resolve_api_predefined() + if api_metric: + self._resolved_metric = api_metric + LazyLoadedPrebuiltMetric._cache[cache_key] = self._resolved_metric + return self._resolved_metric + + # Fallback to GCS loading for custom LLM-based Prebuilt Metrics + logger.debug( + "Metric '%s' not an API Predefined Metric, trying GCS...", self.name + ) + try: + gcs_metric = self._fetch_and_parse(api_client) + final_cache_key = f"{self.name}@{self.version}" + LazyLoadedPrebuiltMetric._cache[final_cache_key] = gcs_metric + self._resolved_metric = gcs_metric + return self._resolved_metric + except Exception as e: + logger.error( + "Error loading metric %s (requested version: %s) from GCS: %s", + self.name, + self.version, + e, + ) + raise ValueError( + f"Metric '{self.name}' could not be resolved as an API " + "Predefined Metric or loaded from GCS." + ) from e + + def __call__( + self, version: Optional[str] = None, **kwargs + ) -> "LazyLoadedPrebuiltMetric": + """Allows setting a specific version and other metric attributes.""" + updated_kwargs = self.metric_kwargs.copy() + updated_kwargs.update(kwargs) + return LazyLoadedPrebuiltMetric( + name=self.name, version=version or self.version, **updated_kwargs + ) + + +class PrebuiltMetricLoader: + """Provides access to predefined evaluation metrics via attributes. + + This class provides a set of predefined LLM-based metrics (Autorater recipes) + for evaluation. These metrics are lazily loaded from a GCS repository + when they are first accessed. + + Example: + from vertexai import types + text_quality_metric = types.RubricMetric.TEXT_QUALITY + """ + + def __getattr__( + self, name: str, version: Optional[str] = None, **kwargs + ) -> LazyLoadedPrebuiltMetric: + return LazyLoadedPrebuiltMetric(name=name, version=version, **kwargs) + + @property + def GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("GENERAL_QUALITY") + + @property + def TEXT_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("TEXT_QUALITY") + + @property + def INSTRUCTION_FOLLOWING(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("INSTRUCTION_FOLLOWING") + + @property + def SAFETY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("SAFETY") + + @property + def MULTI_TURN_GENERAL_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("MULTI_TURN_GENERAL_QUALITY") + + @property + def MULTI_TURN_TEXT_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("MULTI_TURN_TEXT_QUALITY") + + @property + def FINAL_RESPONSE_MATCH(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("FINAL_RESPONSE_MATCH", version="v2") + + @property + def FINAL_RESPONSE_REFERENCE_FREE(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("FINAL_RESPONSE_REFERENCE_FREE") + + @property + def COHERENCE(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("COHERENCE") + + @property + def FLUENCY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("FLUENCY") + + @property + def VERBOSITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("VERBOSITY") + + @property + def SUMMARIZATION_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("SUMMARIZATION_QUALITY") + + @property + def QUESTION_ANSWERING_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("QUESTION_ANSWERING_QUALITY") + + @property + def MULTI_TURN_CHAT_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("MULTI_TURN_CHAT_QUALITY") + + @property + def MULTI_TURN_SAFETY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("MULTI_TURN_SAFETY") + + @property + def FINAL_RESPONSE_QUALITY(self) -> LazyLoadedPrebuiltMetric: + return self.__getattr__("FINAL_RESPONSE_QUALITY") + + +PrebuiltMetric = PrebuiltMetricLoader() +RubricMetric = PrebuiltMetric diff --git a/vertexai/_genai/_evals_utils.py b/vertexai/_genai/_evals_utils.py index 20124a602d..c9aafea1e3 100644 --- a/vertexai/_genai/_evals_utils.py +++ b/vertexai/_genai/_evals_utils.py @@ -15,16 +15,13 @@ """Utility functions for evals.""" import abc -import io import json import logging import os import re -import time from typing import Any, Optional, Union from google.cloud import bigquery -from google.cloud import storage # type: ignore[attr-defined] from google.genai._api_client import BaseApiClient from google.genai._common import get_value_by_path as getv from google.genai._common import set_value_by_path as setv @@ -34,6 +31,7 @@ from . import _evals_constant from . import _transformers from . import types +from . import _gcs_utils logger = logging.getLogger(__name__) @@ -43,168 +41,6 @@ BQ_PREFIX = "bq://" -class GcsUtils: - """Handles File I/O operations with Google Cloud Storage (GCS)""" - - def __init__(self, api_client: BaseApiClient): - self.api_client = api_client - self.storage_client = storage.Client( - project=self.api_client.project, - credentials=self.api_client._credentials, - ) - - def parse_gcs_path(self, gcs_path: str) -> tuple[str, str]: - """Helper to parse gs://bucket/path into (bucket_name, blob_path).""" - if not gcs_path.startswith(GCS_PREFIX): - raise ValueError( - f"Invalid GCS path: '{gcs_path}'. It must start with '{GCS_PREFIX}'." - ) - path_without_prefix = gcs_path[len(GCS_PREFIX) :] - if "/" not in path_without_prefix: - return path_without_prefix, "" - bucket_name, blob_path = path_without_prefix.split("/", 1) - return bucket_name, blob_path - - def upload_file_to_gcs(self, upload_gcs_path: str, filename: str) -> None: - """Uploads the provided file to a Google Cloud Storage location.""" - - storage.Blob.from_string( - uri=upload_gcs_path, client=self.storage_client - ).upload_from_filename(filename) - - def upload_dataframe( - self, - df: "pd.DataFrame", - gcs_destination_blob_path: str, - file_type: str = "jsonl", - ) -> None: - """Uploads a Pandas DataFrame to a Google Cloud Storage location. - - Args: - df: The Pandas DataFrame to upload. - gcs_destination_blob_path: The full GCS path for the destination blob - (e.g., 'gs://bucket/data/my_dataframe.jsonl'). - file_type: The format to save the DataFrame ('jsonl' or 'csv'). Defaults - to 'jsonl'. - """ - bucket_name, blob_name = self.parse_gcs_path(gcs_destination_blob_path) - if not blob_name: - raise ValueError( - f"Invalid GCS path for blob: '{gcs_destination_blob_path}'. " - "It must include the object name (e.g., gs://bucket/file.csv)." - ) - bucket = self.storage_client.bucket(bucket_name) - blob = bucket.blob(blob_name) - - buffer = io.StringIO() - if file_type == "csv": - df.to_csv(buffer, index=False) - content_type = "text/csv" - elif file_type == "jsonl": - df.to_json(buffer, orient="records", lines=True) - content_type = "application/jsonl" - else: - raise ValueError( - f"Unsupported file type: '{file_type}'. " - "Please provide 'jsonl' or 'csv'." - ) - blob.upload_from_string(buffer.getvalue(), content_type=content_type) - - logger.info( - f"DataFrame successfully uploaded to: gs://{bucket.name}/{blob.name}" - ) - - def upload_json(self, data: dict[str, Any], gcs_destination_blob_path: str) -> None: - """Uploads a dictionary as a JSON file to Google Cloud Storage.""" - bucket_name, blob_name = self.parse_gcs_path(gcs_destination_blob_path) - if not blob_name: - raise ValueError( - f"Invalid GCS path for blob: '{gcs_destination_blob_path}'. " - "It must include the object name (e.g., gs://bucket/file.json)." - ) - bucket = self.storage_client.bucket(bucket_name) - blob = bucket.blob(blob_name) - - json_data = json.dumps(data, indent=2) - blob.upload_from_string(json_data, content_type="application/json") - - logger.info( - f"JSON data successfully uploaded to: gs://{bucket_name}/{blob_name}" - ) - - def upload_json_to_prefix( - self, - data: dict[str, Any], - gcs_dest_prefix: str, - filename_prefix: str = "data", - ) -> str: - """Uploads a dictionary to a GCS prefix with a timestamped JSON filename. - - Args: - data: The dictionary to upload. - gcs_dest_prefix: The GCS prefix (e.g., 'gs://bucket/path/prefix/'). - filename_prefix: Prefix for the generated filename. Defaults to 'data'. - - Returns: - The full GCS path where the file was uploaded. - - Raises: - ValueError: If the gcs_dest_prefix is not a valid GCS path. - """ - if not gcs_dest_prefix.startswith(GCS_PREFIX): - raise ValueError( - f"Invalid GCS destination prefix: '{gcs_dest_prefix}'. Must start" - f" with '{GCS_PREFIX}'." - ) - - gcs_path_without_scheme = gcs_dest_prefix[len(GCS_PREFIX) :] - bucket_name, *path_parts = gcs_path_without_scheme.split("/") - - user_prefix_path = "/".join(path_parts) - if user_prefix_path and not user_prefix_path.endswith("/"): - user_prefix_path += "/" - - timestamp = time.strftime("%Y%m%d-%H%M%S") - filename = f"{filename_prefix}_{timestamp}.json" - - blob_name = f"{user_prefix_path}{filename}" - - full_gcs_path = f"{GCS_PREFIX}{bucket_name}/{blob_name}" - - self.upload_json(data, full_gcs_path) - return full_gcs_path - - def read_file_contents(self, gcs_filepath: str) -> Union[str, Any]: - """Reads the contents of a file from Google Cloud Storage.""" - - bucket_name, blob_path = self.parse_gcs_path(gcs_filepath) - if not blob_path: - raise ValueError( - f"Invalid GCS file path: '{gcs_filepath}'. Path must point to a file," - " not just a bucket." - ) - bucket = self.storage_client.bucket(bucket_name) - blob = bucket.blob(blob_path) - content = blob.download_as_bytes().decode("utf-8") - logger.info(f"Successfully read content from '{gcs_filepath}'") - return content - - def read_gcs_file_to_dataframe( - self, gcs_filepath: str, file_type: str - ) -> "pd.DataFrame": - """Reads a file from Google Cloud Storage into a Pandas DataFrame.""" - file_contents = self.read_file_contents(gcs_filepath) - if file_type == "csv": - return pd.read_csv(io.StringIO(file_contents), encoding="utf-8") - elif file_type == "jsonl": - return pd.read_json(io.StringIO(file_contents), lines=True) - else: - raise ValueError( - f"Unsupported file type: '{file_type}'. Please provide 'jsonl' or" - " 'csv'." - ) - - class BigQueryUtils: """Handles BigQuery operations.""" @@ -236,7 +72,7 @@ class EvalDatasetLoader: def __init__(self, api_client: BaseApiClient): self.api_client = api_client - self.gcs_utils = GcsUtils(self.api_client) + self.gcs_utils = _gcs_utils.GcsUtils(self.api_client) self.bigquery_utils = BigQueryUtils(self.api_client) def _load_file( @@ -346,7 +182,7 @@ def _resolve_api_predefined(self) -> Optional[types.Metric]: def _get_latest_version_uri(self, api_client: Any, metric_gcs_dir: str) -> str: """Lists files in GCS directory and determines the latest version URI.""" - gcs_utils = GcsUtils(api_client) + gcs_utils = _gcs_utils.GcsUtils(api_client) bucket_name, prefix = gcs_utils.parse_gcs_path(metric_gcs_dir) blobs = gcs_utils.storage_client.list_blobs(bucket_name, prefix=prefix) @@ -400,7 +236,7 @@ def _fetch_and_parse(self, api_client: Any) -> types.LLMMetric: yaml_uri = os.path.join(metric_gcs_dir, f"{self.version}.yaml") json_uri = os.path.join(metric_gcs_dir, f"{self.version}.json") - gcs_utils = GcsUtils(api_client) + gcs_utils = _gcs_utils.GcsUtils(api_client) try: bucket_name, blob_path = gcs_utils.parse_gcs_path(yaml_uri) if ( @@ -437,7 +273,7 @@ def _fetch_and_parse(self, api_client: Any) -> types.LLMMetric: uri, ) - gcs_utils = GcsUtils(api_client) + gcs_utils = _gcs_utils.GcsUtils(api_client) content_str = gcs_utils.read_file_contents(uri) file_extension = os.path.splitext(uri)[1].lower() @@ -828,7 +664,7 @@ def EvaluateDatasetRequestParameters_to_vertex( @staticmethod def prepare_metric_payload( - request_dict: dict[str, Any], resolved_metrics: list[types.MetricSubclass] + request_dict: dict[str, Any], resolved_metrics: list["types.MetricSubclass"] ) -> dict[str, Any]: """Prepares the metric payload for the evaluation request. @@ -849,6 +685,6 @@ class EvalDataConverter(abc.ABC): """Abstract base class for dataset converters.""" @abc.abstractmethod - def convert(self, raw_data: Any) -> types.EvaluationDataset: + def convert(self, raw_data: Any) -> "types.EvaluationDataset": """Converts a loaded raw dataset into an EvaluationDataset.""" raise NotImplementedError() diff --git a/vertexai/_genai/_evals_visualization.py b/vertexai/_genai/_evals_visualization.py index c327873e22..27705dc0b7 100644 --- a/vertexai/_genai/_evals_visualization.py +++ b/vertexai/_genai/_evals_visualization.py @@ -18,11 +18,12 @@ import logging from typing import Any, Optional -from pydantic import errors import pandas as pd +from pydantic import errors from . import types + logger = logging.getLogger(__name__) diff --git a/vertexai/_genai/_gcs_utils.py b/vertexai/_genai/_gcs_utils.py new file mode 100644 index 0000000000..8e8363dfc8 --- /dev/null +++ b/vertexai/_genai/_gcs_utils.py @@ -0,0 +1,192 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import io +import json +import logging +import time +from typing import Any, Union + +from google.cloud import storage # type: ignore[attr-defined] +from google.genai._api_client import BaseApiClient +import pandas as pd + + +logger = logging.getLogger(__name__) + + +GCS_PREFIX = "gs://" + + +class GcsUtils: + """Handles File I/O operations with Google Cloud Storage (GCS)""" + + def __init__(self, api_client: BaseApiClient): + self.api_client = api_client + self.storage_client = storage.Client( + project=self.api_client.project, + credentials=self.api_client._credentials, + ) + + def parse_gcs_path(self, gcs_path: str) -> tuple[str, str]: + """Helper to parse gs://bucket/path into (bucket_name, blob_path).""" + if not gcs_path.startswith(GCS_PREFIX): + raise ValueError( + f"Invalid GCS path: '{gcs_path}'. It must start with '{GCS_PREFIX}'." + ) + path_without_prefix = gcs_path[len(GCS_PREFIX) :] + if "/" not in path_without_prefix: + return path_without_prefix, "" + bucket_name, blob_path = path_without_prefix.split("/", 1) + return bucket_name, blob_path + + def upload_file_to_gcs(self, upload_gcs_path: str, filename: str) -> None: + """Uploads the provided file to a Google Cloud Storage location.""" + + storage.Blob.from_string( + uri=upload_gcs_path, client=self.storage_client + ).upload_from_filename(filename) + + def upload_dataframe( + self, + df: "pd.DataFrame", + gcs_destination_blob_path: str, + file_type: str = "jsonl", + ) -> None: + """Uploads a Pandas DataFrame to a Google Cloud Storage location. + + Args: + df: The Pandas DataFrame to upload. + gcs_destination_blob_path: The full GCS path for the destination blob + (e.g., 'gs://bucket/data/my_dataframe.jsonl'). + file_type: The format to save the DataFrame ('jsonl' or 'csv'). Defaults + to 'jsonl'. + """ + bucket_name, blob_name = self.parse_gcs_path(gcs_destination_blob_path) + if not blob_name: + raise ValueError( + f"Invalid GCS path for blob: '{gcs_destination_blob_path}'. " + "It must include the object name (e.g., gs://bucket/file.csv)." + ) + bucket = self.storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + buffer = io.StringIO() + if file_type == "csv": + df.to_csv(buffer, index=False) + content_type = "text/csv" + elif file_type == "jsonl": + df.to_json(buffer, orient="records", lines=True) + content_type = "application/jsonl" + else: + raise ValueError( + f"Unsupported file type: '{file_type}'. " + "Please provide 'jsonl' or 'csv'." + ) + blob.upload_from_string(buffer.getvalue(), content_type=content_type) + + logger.info( + f"DataFrame successfully uploaded to: gs://{bucket.name}/{blob.name}" + ) + + def upload_json(self, data: dict[str, Any], gcs_destination_blob_path: str) -> None: + """Uploads a dictionary as a JSON file to Google Cloud Storage.""" + bucket_name, blob_name = self.parse_gcs_path(gcs_destination_blob_path) + if not blob_name: + raise ValueError( + f"Invalid GCS path for blob: '{gcs_destination_blob_path}'. " + "It must include the object name (e.g., gs://bucket/file.json)." + ) + bucket = self.storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + + json_data = json.dumps(data, indent=2) + blob.upload_from_string(json_data, content_type="application/json") + + logger.info( + f"JSON data successfully uploaded to: gs://{bucket_name}/{blob_name}" + ) + + def upload_json_to_prefix( + self, + data: dict[str, Any], + gcs_dest_prefix: str, + filename_prefix: str = "data", + ) -> str: + """Uploads a dictionary to a GCS prefix with a timestamped JSON filename. + + Args: + data: The dictionary to upload. + gcs_dest_prefix: The GCS prefix (e.g., 'gs://bucket/path/prefix/'). + filename_prefix: Prefix for the generated filename. Defaults to 'data'. + + Returns: + The full GCS path where the file was uploaded. + + Raises: + ValueError: If the gcs_dest_prefix is not a valid GCS path. + """ + if not gcs_dest_prefix.startswith(GCS_PREFIX): + raise ValueError( + f"Invalid GCS destination prefix: '{gcs_dest_prefix}'. Must start" + f" with '{GCS_PREFIX}'." + ) + + gcs_path_without_scheme = gcs_dest_prefix[len(GCS_PREFIX) :] + bucket_name, *path_parts = gcs_path_without_scheme.split("/") + + user_prefix_path = "/".join(path_parts) + if user_prefix_path and not user_prefix_path.endswith("/"): + user_prefix_path += "/" + + timestamp = time.strftime("%Y%m%d-%H%M%S") + filename = f"{filename_prefix}_{timestamp}.json" + + blob_name = f"{user_prefix_path}{filename}" + + full_gcs_path = f"{GCS_PREFIX}{bucket_name}/{blob_name}" + + self.upload_json(data, full_gcs_path) + return full_gcs_path + + def read_file_contents(self, gcs_filepath: str) -> Union[str, Any]: + """Reads the contents of a file from Google Cloud Storage.""" + + bucket_name, blob_path = self.parse_gcs_path(gcs_filepath) + if not blob_path: + raise ValueError( + f"Invalid GCS file path: '{gcs_filepath}'. Path must point to a file," + " not just a bucket." + ) + bucket = self.storage_client.bucket(bucket_name) + blob = bucket.blob(blob_path) + content = blob.download_as_bytes().decode("utf-8") + logger.info(f"Successfully read content from '{gcs_filepath}'") + return content + + def read_gcs_file_to_dataframe( + self, gcs_filepath: str, file_type: str + ) -> "pd.DataFrame": + """Reads a file from Google Cloud Storage into a Pandas DataFrame.""" + file_contents = self.read_file_contents(gcs_filepath) + if file_type == "csv": + return pd.read_csv(io.StringIO(file_contents), encoding="utf-8") + elif file_type == "jsonl": + return pd.read_json(io.StringIO(file_contents), lines=True) + else: + raise ValueError( + f"Unsupported file type: '{file_type}'. Please provide 'jsonl' or" + " 'csv'." + ) diff --git a/vertexai/_genai/_transformers.py b/vertexai/_genai/_transformers.py index a5e0cd194c..02356a98be 100644 --- a/vertexai/_genai/_transformers.py +++ b/vertexai/_genai/_transformers.py @@ -23,7 +23,7 @@ def t_metrics( - metrics: list[types.MetricSubclass], + metrics: list["types.MetricSubclass"], set_default_aggregation_metrics: bool = False, ) -> list[dict[str, Any]]: """Prepares the metric payload for the evaluation request. diff --git a/vertexai/_genai/evals.py b/vertexai/_genai/evals.py index f5aa8d6b47..d3c5827468 100644 --- a/vertexai/_genai/evals.py +++ b/vertexai/_genai/evals.py @@ -1323,7 +1323,7 @@ def create_evaluation_run( metrics: Optional[ list[types.EvaluationRunMetricOrDict] ] = None, # TODO: Make required unified metrics available in prod. - agent_info: Optional[types.AgentInfo] = None, + agent_info: Optional[types.evals.AgentInfo] = None, labels: Optional[dict[str, str]] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, ) -> types.EvaluationRun: @@ -2143,7 +2143,7 @@ async def create_evaluation_run( metrics: Optional[ list[types.EvaluationRunMetricOrDict] ] = None, # TODO: Make required unified metrics available in prod. - agent_info: Optional[types.AgentInfo] = None, + agent_info: Optional[types.evals.AgentInfo] = None, labels: Optional[dict[str, str]] = None, config: Optional[types.CreateEvaluationRunConfigOrDict] = None, ) -> types.EvaluationRun: diff --git a/vertexai/_genai/types/__init__.py b/vertexai/_genai/types/__init__.py new file mode 100644 index 0000000000..daa96925a6 --- /dev/null +++ b/vertexai/_genai/types/__init__.py @@ -0,0 +1,1855 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Code generated by the Google Gen AI SDK generator DO NOT EDIT. +# flake8: noqa: F401 + +from . import agent_engines +from . import evals +from .common import _AppendAgentEngineSessionEventRequestParameters +from .common import _CreateAgentEngineMemoryRequestParameters +from .common import _CreateAgentEngineRequestParameters +from .common import _CreateAgentEngineSandboxRequestParameters +from .common import _CreateAgentEngineSessionRequestParameters +from .common import _CreateDatasetParameters +from .common import _CreateDatasetVersionParameters +from .common import _CreateEvaluationItemParameters +from .common import _CreateEvaluationRunParameters +from .common import _CreateEvaluationSetParameters +from .common import _CreateMultimodalDatasetParameters +from .common import _CustomJobParameters +from .common import _DeleteAgentEngineMemoryRequestParameters +from .common import _DeleteAgentEngineRequestParameters +from .common import _DeleteAgentEngineSandboxRequestParameters +from .common import _DeleteAgentEngineSessionRequestParameters +from .common import _DeleteDatasetRequestParameters +from .common import _DeletePromptVersionRequestParameters +from .common import _EvaluateInstancesRequestParameters +from .common import _ExecuteCodeAgentEngineSandboxRequestParameters +from .common import _GenerateAgentEngineMemoriesRequestParameters +from .common import _GenerateInstanceRubricsRequest +from .common import _GetAgentEngineGenerateMemoriesOperationParameters +from .common import _GetAgentEngineMemoryOperationParameters +from .common import _GetAgentEngineMemoryRequestParameters +from .common import _GetAgentEngineMemoryRevisionRequestParameters +from .common import _GetAgentEngineOperationParameters +from .common import _GetAgentEngineRequestParameters +from .common import _GetAgentEngineSandboxOperationParameters +from .common import _GetAgentEngineSandboxRequestParameters +from .common import _GetAgentEngineSessionOperationParameters +from .common import _GetAgentEngineSessionRequestParameters +from .common import _GetCustomJobParameters +from .common import _GetDatasetOperationParameters +from .common import _GetDatasetParameters +from .common import _GetDatasetVersionParameters +from .common import _GetEvaluationItemParameters +from .common import _GetEvaluationRunParameters +from .common import _GetEvaluationSetParameters +from .common import _GetMultimodalDatasetOperationParameters +from .common import _ListAgentEngineMemoryRequestParameters +from .common import _ListAgentEngineMemoryRevisionsRequestParameters +from .common import _ListAgentEngineRequestParameters +from .common import _ListAgentEngineSandboxesRequestParameters +from .common import _ListAgentEngineSessionEventsRequestParameters +from .common import _ListAgentEngineSessionsRequestParameters +from .common import _ListDatasetsRequestParameters +from .common import _ListDatasetVersionsRequestParameters +from .common import _OptimizeRequestParameters +from .common import _QueryAgentEngineRequestParameters +from .common import _RestoreVersionRequestParameters +from .common import _RetrieveAgentEngineMemoriesRequestParameters +from .common import _RollbackAgentEngineMemoryRequestParameters +from .common import _UpdateAgentEngineMemoryRequestParameters +from .common import _UpdateAgentEngineRequestParameters +from .common import _UpdateAgentEngineSessionRequestParameters +from .common import _UpdateDatasetParameters +from .common import AcceleratorType +from .common import AgentEngine +from .common import AgentEngineConfig +from .common import AgentEngineConfigDict +from .common import AgentEngineConfigOrDict +from .common import AgentEngineDict +from .common import AgentEngineGenerateMemoriesOperation +from .common import AgentEngineGenerateMemoriesOperationDict +from .common import AgentEngineGenerateMemoriesOperationOrDict +from .common import AgentEngineMemoryConfig +from .common import AgentEngineMemoryConfigDict +from .common import AgentEngineMemoryConfigOrDict +from .common import AgentEngineMemoryOperation +from .common import AgentEngineMemoryOperationDict +from .common import AgentEngineMemoryOperationOrDict +from .common import AgentEngineOperation +from .common import AgentEngineOperationDict +from .common import AgentEngineOperationOrDict +from .common import AgentEngineOrDict +from .common import AgentEngineRollbackMemoryOperation +from .common import AgentEngineRollbackMemoryOperationDict +from .common import AgentEngineRollbackMemoryOperationOrDict +from .common import AgentEngineSandboxOperation +from .common import AgentEngineSandboxOperationDict +from .common import AgentEngineSandboxOperationOrDict +from .common import AgentEngineSessionOperation +from .common import AgentEngineSessionOperationDict +from .common import AgentEngineSessionOperationOrDict +from .common import AgentServerMode +from .common import AggregatedMetricResult +from .common import AggregatedMetricResultDict +from .common import AggregatedMetricResultOrDict +from .common import AppendAgentEngineSessionEventConfig +from .common import AppendAgentEngineSessionEventConfigDict +from .common import AppendAgentEngineSessionEventConfigOrDict +from .common import AppendAgentEngineSessionEventResponse +from .common import AppendAgentEngineSessionEventResponseDict +from .common import AppendAgentEngineSessionEventResponseOrDict +from .common import ApplicableGuideline +from .common import ApplicableGuidelineDict +from .common import ApplicableGuidelineOrDict +from .common import BigQueryRequestSet +from .common import BigQueryRequestSetDict +from .common import BigQueryRequestSetOrDict +from .common import BigQuerySource +from .common import BigQuerySourceDict +from .common import BigQuerySourceOrDict +from .common import BleuInput +from .common import BleuInputDict +from .common import BleuInputOrDict +from .common import BleuInstance +from .common import BleuInstanceDict +from .common import BleuInstanceOrDict +from .common import BleuMetricValue +from .common import BleuMetricValueDict +from .common import BleuMetricValueOrDict +from .common import BleuResults +from .common import BleuResultsDict +from .common import BleuResultsOrDict +from .common import CandidateResponse +from .common import CandidateResponseDict +from .common import CandidateResponseOrDict +from .common import CandidateResult +from .common import CandidateResultDict +from .common import CandidateResultOrDict +from .common import Chunk +from .common import ChunkDict +from .common import ChunkOrDict +from .common import CometResult +from .common import CometResultDict +from .common import CometResultOrDict +from .common import ContainerSpec +from .common import ContainerSpecDict +from .common import ContainerSpecOrDict +from .common import ContentMap +from .common import ContentMapContents +from .common import ContentMapContentsDict +from .common import ContentMapContentsOrDict +from .common import ContentMapDict +from .common import ContentMapOrDict +from .common import CreateAgentEngineConfig +from .common import CreateAgentEngineConfigDict +from .common import CreateAgentEngineConfigOrDict +from .common import CreateAgentEngineSandboxConfig +from .common import CreateAgentEngineSandboxConfigDict +from .common import CreateAgentEngineSandboxConfigOrDict +from .common import CreateAgentEngineSessionConfig +from .common import CreateAgentEngineSessionConfigDict +from .common import CreateAgentEngineSessionConfigOrDict +from .common import CreateDatasetConfig +from .common import CreateDatasetConfigDict +from .common import CreateDatasetConfigOrDict +from .common import CreateDatasetVersionConfig +from .common import CreateDatasetVersionConfigDict +from .common import CreateDatasetVersionConfigOrDict +from .common import CreateEvaluationItemConfig +from .common import CreateEvaluationItemConfigDict +from .common import CreateEvaluationItemConfigOrDict +from .common import CreateEvaluationRunConfig +from .common import CreateEvaluationRunConfigDict +from .common import CreateEvaluationRunConfigOrDict +from .common import CreateEvaluationSetConfig +from .common import CreateEvaluationSetConfigDict +from .common import CreateEvaluationSetConfigOrDict +from .common import CreateMultimodalDatasetConfig +from .common import CreateMultimodalDatasetConfigDict +from .common import CreateMultimodalDatasetConfigOrDict +from .common import CreatePromptConfig +from .common import CreatePromptConfigDict +from .common import CreatePromptConfigOrDict +from .common import CreatePromptVersionConfig +from .common import CreatePromptVersionConfigDict +from .common import CreatePromptVersionConfigOrDict +from .common import CustomJob +from .common import CustomJobDict +from .common import CustomJobOrDict +from .common import CustomJobSpec +from .common import CustomJobSpecDict +from .common import CustomJobSpecOrDict +from .common import CustomOutput +from .common import CustomOutputDict +from .common import CustomOutputOrDict +from .common import Dataset +from .common import DatasetDict +from .common import DatasetOperation +from .common import DatasetOperationDict +from .common import DatasetOperationOrDict +from .common import DatasetOrDict +from .common import DatasetVersion +from .common import DatasetVersionDict +from .common import DatasetVersionOrDict +from .common import DeleteAgentEngineConfig +from .common import DeleteAgentEngineConfigDict +from .common import DeleteAgentEngineConfigOrDict +from .common import DeleteAgentEngineMemoryConfig +from .common import DeleteAgentEngineMemoryConfigDict +from .common import DeleteAgentEngineMemoryConfigOrDict +from .common import DeleteAgentEngineMemoryOperation +from .common import DeleteAgentEngineMemoryOperationDict +from .common import DeleteAgentEngineMemoryOperationOrDict +from .common import DeleteAgentEngineOperation +from .common import DeleteAgentEngineOperationDict +from .common import DeleteAgentEngineOperationOrDict +from .common import DeleteAgentEngineSandboxConfig +from .common import DeleteAgentEngineSandboxConfigDict +from .common import DeleteAgentEngineSandboxConfigOrDict +from .common import DeleteAgentEngineSandboxOperation +from .common import DeleteAgentEngineSandboxOperationDict +from .common import DeleteAgentEngineSandboxOperationOrDict +from .common import DeleteAgentEngineSessionConfig +from .common import DeleteAgentEngineSessionConfigDict +from .common import DeleteAgentEngineSessionConfigOrDict +from .common import DeleteAgentEngineSessionOperation +from .common import DeleteAgentEngineSessionOperationDict +from .common import DeleteAgentEngineSessionOperationOrDict +from .common import DeletePromptConfig +from .common import DeletePromptConfigDict +from .common import DeletePromptConfigOrDict +from .common import DeletePromptOperation +from .common import DeletePromptOperationDict +from .common import DeletePromptOperationOrDict +from .common import DeletePromptVersionOperation +from .common import DeletePromptVersionOperationDict +from .common import DeletePromptVersionOperationOrDict +from .common import DiskSpec +from .common import DiskSpecDict +from .common import DiskSpecOrDict +from .common import DnsPeeringConfig +from .common import DnsPeeringConfigDict +from .common import DnsPeeringConfigOrDict +from .common import EnvVar +from .common import EnvVarDict +from .common import EnvVarOrDict +from .common import EvalCase +from .common import EvalCaseDict +from .common import EvalCaseMetricResult +from .common import EvalCaseMetricResultDict +from .common import EvalCaseMetricResultOrDict +from .common import EvalCaseOrDict +from .common import EvalCaseResult +from .common import EvalCaseResultDict +from .common import EvalCaseResultOrDict +from .common import EvalRunInferenceConfig +from .common import EvalRunInferenceConfigDict +from .common import EvalRunInferenceConfigOrDict +from .common import EvaluateDatasetConfig +from .common import EvaluateDatasetConfigDict +from .common import EvaluateDatasetConfigOrDict +from .common import EvaluateDatasetOperation +from .common import EvaluateDatasetOperationDict +from .common import EvaluateDatasetOperationOrDict +from .common import EvaluateDatasetRequestParameters +from .common import EvaluateDatasetRequestParametersDict +from .common import EvaluateDatasetRequestParametersOrDict +from .common import EvaluateInstancesConfig +from .common import EvaluateInstancesConfigDict +from .common import EvaluateInstancesConfigOrDict +from .common import EvaluateInstancesResponse +from .common import EvaluateInstancesResponseDict +from .common import EvaluateInstancesResponseOrDict +from .common import EvaluateMethodConfig +from .common import EvaluateMethodConfigDict +from .common import EvaluateMethodConfigOrDict +from .common import EvaluationDataset +from .common import EvaluationDatasetDict +from .common import EvaluationDatasetOrDict +from .common import EvaluationInstance +from .common import EvaluationInstanceDict +from .common import EvaluationInstanceOrDict +from .common import EvaluationItem +from .common import EvaluationItemDict +from .common import EvaluationItemOrDict +from .common import EvaluationItemRequest +from .common import EvaluationItemRequestDict +from .common import EvaluationItemRequestOrDict +from .common import EvaluationItemResult +from .common import EvaluationItemResultDict +from .common import EvaluationItemResultOrDict +from .common import EvaluationItemType +from .common import EvaluationPrompt +from .common import EvaluationPromptDict +from .common import EvaluationPromptOrDict +from .common import EvaluationResult +from .common import EvaluationResultDict +from .common import EvaluationResultOrDict +from .common import EvaluationRun +from .common import EvaluationRunAgentConfig +from .common import EvaluationRunAgentConfigDict +from .common import EvaluationRunAgentConfigOrDict +from .common import EvaluationRunConfig +from .common import EvaluationRunConfigDict +from .common import EvaluationRunConfigOrDict +from .common import EvaluationRunDataSource +from .common import EvaluationRunDataSourceDict +from .common import EvaluationRunDataSourceOrDict +from .common import EvaluationRunDict +from .common import EvaluationRunInferenceConfig +from .common import EvaluationRunInferenceConfigDict +from .common import EvaluationRunInferenceConfigOrDict +from .common import EvaluationRunMetadata +from .common import EvaluationRunMetadataDict +from .common import EvaluationRunMetadataOrDict +from .common import EvaluationRunMetric +from .common import EvaluationRunMetricDict +from .common import EvaluationRunMetricOrDict +from .common import EvaluationRunOrDict +from .common import EvaluationRunResults +from .common import EvaluationRunResultsDict +from .common import EvaluationRunResultsOrDict +from .common import EvaluationRunState +from .common import EvaluationSet +from .common import EvaluationSetDict +from .common import EvaluationSetOrDict +from .common import Event +from .common import EventActions +from .common import EventActionsDict +from .common import EventActionsOrDict +from .common import EventDict +from .common import EventMetadata +from .common import EventMetadataDict +from .common import EventMetadataOrDict +from .common import EventOrDict +from .common import ExactMatchInput +from .common import ExactMatchInputDict +from .common import ExactMatchInputOrDict +from .common import ExactMatchInstance +from .common import ExactMatchInstanceDict +from .common import ExactMatchInstanceOrDict +from .common import ExactMatchMetricValue +from .common import ExactMatchMetricValueDict +from .common import ExactMatchMetricValueOrDict +from .common import ExactMatchResults +from .common import ExactMatchResultsDict +from .common import ExactMatchResultsOrDict +from .common import ExactMatchSpec +from .common import ExactMatchSpecDict +from .common import ExactMatchSpecOrDict +from .common import ExecuteCodeAgentEngineSandboxConfig +from .common import ExecuteCodeAgentEngineSandboxConfigDict +from .common import ExecuteCodeAgentEngineSandboxConfigOrDict +from .common import ExecuteSandboxEnvironmentResponse +from .common import ExecuteSandboxEnvironmentResponseDict +from .common import ExecuteSandboxEnvironmentResponseOrDict +from .common import GcsSource +from .common import GcsSourceDict +from .common import GcsSourceOrDict +from .common import GenerateAgentEngineMemoriesConfig +from .common import GenerateAgentEngineMemoriesConfigDict +from .common import GenerateAgentEngineMemoriesConfigOrDict +from .common import GenerateInstanceRubricsResponse +from .common import GenerateInstanceRubricsResponseDict +from .common import GenerateInstanceRubricsResponseOrDict +from .common import GenerateMemoriesRequestDirectContentsSource +from .common import GenerateMemoriesRequestDirectContentsSourceDict +from .common import GenerateMemoriesRequestDirectContentsSourceEvent +from .common import GenerateMemoriesRequestDirectContentsSourceEventDict +from .common import GenerateMemoriesRequestDirectContentsSourceEventOrDict +from .common import GenerateMemoriesRequestDirectContentsSourceOrDict +from .common import GenerateMemoriesRequestDirectMemoriesSource +from .common import GenerateMemoriesRequestDirectMemoriesSourceDict +from .common import GenerateMemoriesRequestDirectMemoriesSourceDirectMemory +from .common import GenerateMemoriesRequestDirectMemoriesSourceDirectMemoryDict +from .common import GenerateMemoriesRequestDirectMemoriesSourceDirectMemoryOrDict +from .common import GenerateMemoriesRequestDirectMemoriesSourceOrDict +from .common import GenerateMemoriesRequestVertexSessionSource +from .common import GenerateMemoriesRequestVertexSessionSourceDict +from .common import GenerateMemoriesRequestVertexSessionSourceOrDict +from .common import GenerateMemoriesResponse +from .common import GenerateMemoriesResponseDict +from .common import GenerateMemoriesResponseGeneratedMemory +from .common import GenerateMemoriesResponseGeneratedMemoryAction +from .common import GenerateMemoriesResponseGeneratedMemoryDict +from .common import GenerateMemoriesResponseGeneratedMemoryOrDict +from .common import GenerateMemoriesResponseOrDict +from .common import GetAgentEngineConfig +from .common import GetAgentEngineConfigDict +from .common import GetAgentEngineConfigOrDict +from .common import GetAgentEngineMemoryConfig +from .common import GetAgentEngineMemoryConfigDict +from .common import GetAgentEngineMemoryConfigOrDict +from .common import GetAgentEngineMemoryRevisionConfig +from .common import GetAgentEngineMemoryRevisionConfigDict +from .common import GetAgentEngineMemoryRevisionConfigOrDict +from .common import GetAgentEngineOperationConfig +from .common import GetAgentEngineOperationConfigDict +from .common import GetAgentEngineOperationConfigOrDict +from .common import GetAgentEngineSandboxConfig +from .common import GetAgentEngineSandboxConfigDict +from .common import GetAgentEngineSandboxConfigOrDict +from .common import GetAgentEngineSessionConfig +from .common import GetAgentEngineSessionConfigDict +from .common import GetAgentEngineSessionConfigOrDict +from .common import GetDatasetOperationConfig +from .common import GetDatasetOperationConfigDict +from .common import GetDatasetOperationConfigOrDict +from .common import GetEvaluationItemConfig +from .common import GetEvaluationItemConfigDict +from .common import GetEvaluationItemConfigOrDict +from .common import GetEvaluationRunConfig +from .common import GetEvaluationRunConfigDict +from .common import GetEvaluationRunConfigOrDict +from .common import GetEvaluationSetConfig +from .common import GetEvaluationSetConfigDict +from .common import GetEvaluationSetConfigOrDict +from .common import GetMultimodalDatasetOperationConfig +from .common import GetMultimodalDatasetOperationConfigDict +from .common import GetMultimodalDatasetOperationConfigOrDict +from .common import GetPromptConfig +from .common import GetPromptConfigDict +from .common import GetPromptConfigOrDict +from .common import Importance +from .common import JobState +from .common import Language +from .common import ListAgentEngineConfig +from .common import ListAgentEngineConfigDict +from .common import ListAgentEngineConfigOrDict +from .common import ListAgentEngineMemoryConfig +from .common import ListAgentEngineMemoryConfigDict +from .common import ListAgentEngineMemoryConfigOrDict +from .common import ListAgentEngineMemoryRevisionsConfig +from .common import ListAgentEngineMemoryRevisionsConfigDict +from .common import ListAgentEngineMemoryRevisionsConfigOrDict +from .common import ListAgentEngineMemoryRevisionsResponse +from .common import ListAgentEngineMemoryRevisionsResponseDict +from .common import ListAgentEngineMemoryRevisionsResponseOrDict +from .common import ListAgentEngineSandboxesConfig +from .common import ListAgentEngineSandboxesConfigDict +from .common import ListAgentEngineSandboxesConfigOrDict +from .common import ListAgentEngineSandboxesResponse +from .common import ListAgentEngineSandboxesResponseDict +from .common import ListAgentEngineSandboxesResponseOrDict +from .common import ListAgentEngineSessionEventsConfig +from .common import ListAgentEngineSessionEventsConfigDict +from .common import ListAgentEngineSessionEventsConfigOrDict +from .common import ListAgentEngineSessionEventsResponse +from .common import ListAgentEngineSessionEventsResponseDict +from .common import ListAgentEngineSessionEventsResponseOrDict +from .common import ListAgentEngineSessionsConfig +from .common import ListAgentEngineSessionsConfigDict +from .common import ListAgentEngineSessionsConfigOrDict +from .common import ListDatasetsResponse +from .common import ListDatasetsResponseDict +from .common import ListDatasetsResponseOrDict +from .common import ListDatasetVersionsResponse +from .common import ListDatasetVersionsResponseDict +from .common import ListDatasetVersionsResponseOrDict +from .common import ListPromptsConfig +from .common import ListPromptsConfigDict +from .common import ListPromptsConfigOrDict +from .common import ListReasoningEnginesMemoriesResponse +from .common import ListReasoningEnginesMemoriesResponseDict +from .common import ListReasoningEnginesMemoriesResponseOrDict +from .common import ListReasoningEnginesResponse +from .common import ListReasoningEnginesResponseDict +from .common import ListReasoningEnginesResponseOrDict +from .common import ListReasoningEnginesSessionsResponse +from .common import ListReasoningEnginesSessionsResponseDict +from .common import ListReasoningEnginesSessionsResponseOrDict +from .common import LLMBasedMetricSpec +from .common import LLMBasedMetricSpecDict +from .common import LLMBasedMetricSpecOrDict +from .common import LLMMetric +from .common import MachineConfig +from .common import MachineSpec +from .common import MachineSpecDict +from .common import MachineSpecOrDict +from .common import ManagedTopicEnum +from .common import MapInstance +from .common import MapInstanceDict +from .common import MapInstanceOrDict +from .common import Memory +from .common import MemoryBankCustomizationConfig +from .common import MemoryBankCustomizationConfigDict +from .common import MemoryBankCustomizationConfigGenerateMemoriesExample +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSource, +) +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceDict, +) +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceEvent, +) +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceEventDict, +) +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceEventOrDict, +) +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceOrDict, +) +from .common import MemoryBankCustomizationConfigGenerateMemoriesExampleDict +from .common import MemoryBankCustomizationConfigGenerateMemoriesExampleGeneratedMemory +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleGeneratedMemoryDict, +) +from .common import ( + MemoryBankCustomizationConfigGenerateMemoriesExampleGeneratedMemoryOrDict, +) +from .common import MemoryBankCustomizationConfigGenerateMemoriesExampleOrDict +from .common import MemoryBankCustomizationConfigMemoryTopic +from .common import MemoryBankCustomizationConfigMemoryTopicCustomMemoryTopic +from .common import MemoryBankCustomizationConfigMemoryTopicCustomMemoryTopicDict +from .common import MemoryBankCustomizationConfigMemoryTopicCustomMemoryTopicOrDict +from .common import MemoryBankCustomizationConfigMemoryTopicDict +from .common import MemoryBankCustomizationConfigMemoryTopicManagedMemoryTopic +from .common import MemoryBankCustomizationConfigMemoryTopicManagedMemoryTopicDict +from .common import MemoryBankCustomizationConfigMemoryTopicManagedMemoryTopicOrDict +from .common import MemoryBankCustomizationConfigMemoryTopicOrDict +from .common import MemoryBankCustomizationConfigOrDict +from .common import MemoryDict +from .common import MemoryOrDict +from .common import MemoryRevision +from .common import MemoryRevisionDict +from .common import MemoryRevisionOrDict +from .common import MemoryTopicId +from .common import MemoryTopicIdDict +from .common import MemoryTopicIdOrDict +from .common import Message +from .common import MessageDict +from .common import MessageOrDict +from .common import Metadata +from .common import MetadataDict +from .common import MetadataOrDict +from .common import Metric +from .common import MetricDict +from .common import MetricOrDict +from .common import MetricPromptBuilder +from .common import MetricResult +from .common import MetricResultDict +from .common import MetricResultOrDict +from .common import MetricxResult +from .common import MetricxResultDict +from .common import MetricxResultOrDict +from .common import MultimodalDataset +from .common import MultimodalDatasetDict +from .common import MultimodalDatasetOperation +from .common import MultimodalDatasetOperationDict +from .common import MultimodalDatasetOperationOrDict +from .common import MultimodalDatasetOrDict +from .common import NfsMount +from .common import NfsMountDict +from .common import NfsMountOrDict +from .common import ObservabilityEvalCase +from .common import ObservabilityEvalCaseDict +from .common import ObservabilityEvalCaseOrDict +from .common import OptimizeConfig +from .common import OptimizeConfigDict +from .common import OptimizeConfigOrDict +from .common import OptimizeResponse +from .common import OptimizeResponseDict +from .common import OptimizeResponseEndpoint +from .common import OptimizeResponseEndpointDict +from .common import OptimizeResponseEndpointOrDict +from .common import OptimizeResponseOrDict +from .common import PairwiseChoice +from .common import PairwiseMetricInput +from .common import PairwiseMetricInputDict +from .common import PairwiseMetricInputOrDict +from .common import PairwiseMetricInstance +from .common import PairwiseMetricInstanceDict +from .common import PairwiseMetricInstanceOrDict +from .common import PairwiseMetricResult +from .common import PairwiseMetricResultDict +from .common import PairwiseMetricResultOrDict +from .common import ParsedResponse +from .common import ParsedResponseDict +from .common import ParsedResponseOrDict +from .common import PointwiseMetricInput +from .common import PointwiseMetricInputDict +from .common import PointwiseMetricInputOrDict +from .common import PointwiseMetricInstance +from .common import PointwiseMetricInstanceDict +from .common import PointwiseMetricInstanceOrDict +from .common import PointwiseMetricResult +from .common import PointwiseMetricResultDict +from .common import PointwiseMetricResultOrDict +from .common import PredefinedMetricSpec +from .common import PredefinedMetricSpecDict +from .common import PredefinedMetricSpecOrDict +from .common import Prompt +from .common import PromptData +from .common import PromptDataDict +from .common import PromptDataOrDict +from .common import PromptDict +from .common import PromptOptimizerVAPOConfig +from .common import PromptOptimizerVAPOConfigDict +from .common import PromptOptimizerVAPOConfigOrDict +from .common import PromptOrDict +from .common import PromptRef +from .common import PromptRefDict +from .common import PromptRefOrDict +from .common import PromptTemplate +from .common import PromptTemplateData +from .common import PromptTemplateDataDict +from .common import PromptTemplateDataOrDict +from .common import PromptTemplateDict +from .common import PromptTemplateOrDict +from .common import PromptVersionRef +from .common import PromptVersionRefDict +from .common import PromptVersionRefOrDict +from .common import PscInterfaceConfig +from .common import PscInterfaceConfigDict +from .common import PscInterfaceConfigOrDict +from .common import PythonPackageSpec +from .common import PythonPackageSpecDict +from .common import PythonPackageSpecOrDict +from .common import QueryAgentEngineConfig +from .common import QueryAgentEngineConfigDict +from .common import QueryAgentEngineConfigOrDict +from .common import QueryReasoningEngineResponse +from .common import QueryReasoningEngineResponseDict +from .common import QueryReasoningEngineResponseOrDict +from .common import RawOutput +from .common import RawOutputDict +from .common import RawOutputOrDict +from .common import ReasoningEngine +from .common import ReasoningEngineContextSpec +from .common import ReasoningEngineContextSpecDict +from .common import ReasoningEngineContextSpecMemoryBankConfig +from .common import ReasoningEngineContextSpecMemoryBankConfigDict +from .common import ReasoningEngineContextSpecMemoryBankConfigGenerationConfig +from .common import ReasoningEngineContextSpecMemoryBankConfigGenerationConfigDict +from .common import ReasoningEngineContextSpecMemoryBankConfigGenerationConfigOrDict +from .common import ReasoningEngineContextSpecMemoryBankConfigOrDict +from .common import ReasoningEngineContextSpecMemoryBankConfigSimilaritySearchConfig +from .common import ReasoningEngineContextSpecMemoryBankConfigSimilaritySearchConfigDict +from .common import ( + ReasoningEngineContextSpecMemoryBankConfigSimilaritySearchConfigOrDict, +) +from .common import ReasoningEngineContextSpecMemoryBankConfigTtlConfig +from .common import ReasoningEngineContextSpecMemoryBankConfigTtlConfigDict +from .common import ReasoningEngineContextSpecMemoryBankConfigTtlConfigGranularTtlConfig +from .common import ( + ReasoningEngineContextSpecMemoryBankConfigTtlConfigGranularTtlConfigDict, +) +from .common import ( + ReasoningEngineContextSpecMemoryBankConfigTtlConfigGranularTtlConfigOrDict, +) +from .common import ReasoningEngineContextSpecMemoryBankConfigTtlConfigOrDict +from .common import ReasoningEngineContextSpecOrDict +from .common import ReasoningEngineDict +from .common import ReasoningEngineOrDict +from .common import ReasoningEngineSpec +from .common import ReasoningEngineSpecDeploymentSpec +from .common import ReasoningEngineSpecDeploymentSpecDict +from .common import ReasoningEngineSpecDeploymentSpecOrDict +from .common import ReasoningEngineSpecDict +from .common import ReasoningEngineSpecOrDict +from .common import ReasoningEngineSpecPackageSpec +from .common import ReasoningEngineSpecPackageSpecDict +from .common import ReasoningEngineSpecPackageSpecOrDict +from .common import ReservationAffinity +from .common import ReservationAffinityDict +from .common import ReservationAffinityOrDict +from .common import ResponseCandidate +from .common import ResponseCandidateDict +from .common import ResponseCandidateOrDict +from .common import ResponseCandidateResult +from .common import ResponseCandidateResultDict +from .common import ResponseCandidateResultOrDict +from .common import RestoreVersionConfig +from .common import RestoreVersionConfigDict +from .common import RestoreVersionConfigOrDict +from .common import RestoreVersionOperation +from .common import RestoreVersionOperationDict +from .common import RestoreVersionOperationOrDict +from .common import RetrieveAgentEngineMemoriesConfig +from .common import RetrieveAgentEngineMemoriesConfigDict +from .common import RetrieveAgentEngineMemoriesConfigOrDict +from .common import RetrieveMemoriesRequestSimilaritySearchParams +from .common import RetrieveMemoriesRequestSimilaritySearchParamsDict +from .common import RetrieveMemoriesRequestSimilaritySearchParamsOrDict +from .common import RetrieveMemoriesRequestSimpleRetrievalParams +from .common import RetrieveMemoriesRequestSimpleRetrievalParamsDict +from .common import RetrieveMemoriesRequestSimpleRetrievalParamsOrDict +from .common import RetrieveMemoriesResponse +from .common import RetrieveMemoriesResponseDict +from .common import RetrieveMemoriesResponseOrDict +from .common import RetrieveMemoriesResponseRetrievedMemory +from .common import RetrieveMemoriesResponseRetrievedMemoryDict +from .common import RetrieveMemoriesResponseRetrievedMemoryOrDict +from .common import RollbackAgentEngineMemoryConfig +from .common import RollbackAgentEngineMemoryConfigDict +from .common import RollbackAgentEngineMemoryConfigOrDict +from .common import RougeInput +from .common import RougeInputDict +from .common import RougeInputOrDict +from .common import RougeInstance +from .common import RougeInstanceDict +from .common import RougeInstanceOrDict +from .common import RougeMetricValue +from .common import RougeMetricValueDict +from .common import RougeMetricValueOrDict +from .common import RougeResults +from .common import RougeResultsDict +from .common import RougeResultsOrDict +from .common import Rubric +from .common import RubricBasedMetricInput +from .common import RubricBasedMetricInputDict +from .common import RubricBasedMetricInputOrDict +from .common import RubricBasedMetricInstance +from .common import RubricBasedMetricInstanceDict +from .common import RubricBasedMetricInstanceOrDict +from .common import RubricBasedMetricResult +from .common import RubricBasedMetricResultDict +from .common import RubricBasedMetricResultOrDict +from .common import RubricBasedMetricSpec +from .common import RubricBasedMetricSpecDict +from .common import RubricBasedMetricSpecOrDict +from .common import RubricContent +from .common import RubricContentDict +from .common import RubricContentOrDict +from .common import RubricContentProperty +from .common import RubricContentPropertyDict +from .common import RubricContentPropertyOrDict +from .common import RubricContentType +from .common import RubricDict +from .common import RubricEnhancedContents +from .common import RubricEnhancedContentsDict +from .common import RubricEnhancedContentsOrDict +from .common import RubricGenerationConfig +from .common import RubricGenerationConfigDict +from .common import RubricGenerationConfigOrDict +from .common import RubricGenerationSpec +from .common import RubricGenerationSpecDict +from .common import RubricGenerationSpecOrDict +from .common import RubricGroup +from .common import RubricGroupDict +from .common import RubricGroupOrDict +from .common import RubricMetric +from .common import RubricOrDict +from .common import RubricVerdict +from .common import RubricVerdictDict +from .common import RubricVerdictOrDict +from .common import SamplingConfig +from .common import SamplingConfigDict +from .common import SamplingConfigOrDict +from .common import SamplingMethod +from .common import SandboxEnvironment +from .common import SandboxEnvironmentConnectionInfo +from .common import SandboxEnvironmentConnectionInfoDict +from .common import SandboxEnvironmentConnectionInfoOrDict +from .common import SandboxEnvironmentDict +from .common import SandboxEnvironmentOrDict +from .common import SandboxEnvironmentSpec +from .common import SandboxEnvironmentSpecCodeExecutionEnvironment +from .common import SandboxEnvironmentSpecCodeExecutionEnvironmentDict +from .common import SandboxEnvironmentSpecCodeExecutionEnvironmentOrDict +from .common import SandboxEnvironmentSpecComputerUseEnvironment +from .common import SandboxEnvironmentSpecComputerUseEnvironmentDict +from .common import SandboxEnvironmentSpecComputerUseEnvironmentOrDict +from .common import SandboxEnvironmentSpecDict +from .common import SandboxEnvironmentSpecOrDict +from .common import SavedQuery +from .common import SavedQueryDict +from .common import SavedQueryOrDict +from .common import Scheduling +from .common import SchedulingDict +from .common import SchedulingOrDict +from .common import SchemaPredictParamsGroundingConfig +from .common import SchemaPredictParamsGroundingConfigDict +from .common import SchemaPredictParamsGroundingConfigOrDict +from .common import SchemaPredictParamsGroundingConfigSourceEntry +from .common import SchemaPredictParamsGroundingConfigSourceEntryDict +from .common import SchemaPredictParamsGroundingConfigSourceEntryOrDict +from .common import SchemaPromptApiSchema +from .common import SchemaPromptApiSchemaDict +from .common import SchemaPromptApiSchemaOrDict +from .common import SchemaPromptInstancePromptExecution +from .common import SchemaPromptInstancePromptExecutionDict +from .common import SchemaPromptInstancePromptExecutionOrDict +from .common import SchemaPromptInstanceVariableValue +from .common import SchemaPromptInstanceVariableValueDict +from .common import SchemaPromptInstanceVariableValueOrDict +from .common import SchemaPromptSpecMultimodalPrompt +from .common import SchemaPromptSpecMultimodalPromptDict +from .common import SchemaPromptSpecMultimodalPromptOrDict +from .common import SchemaPromptSpecPartList +from .common import SchemaPromptSpecPartListDict +from .common import SchemaPromptSpecPartListOrDict +from .common import SchemaPromptSpecPromptMessage +from .common import SchemaPromptSpecPromptMessageDict +from .common import SchemaPromptSpecPromptMessageOrDict +from .common import SchemaPromptSpecReferenceSentencePair +from .common import SchemaPromptSpecReferenceSentencePairDict +from .common import SchemaPromptSpecReferenceSentencePairList +from .common import SchemaPromptSpecReferenceSentencePairListDict +from .common import SchemaPromptSpecReferenceSentencePairListOrDict +from .common import SchemaPromptSpecReferenceSentencePairOrDict +from .common import SchemaPromptSpecStructuredPrompt +from .common import SchemaPromptSpecStructuredPromptDict +from .common import SchemaPromptSpecStructuredPromptOrDict +from .common import SchemaPromptSpecTranslationExample +from .common import SchemaPromptSpecTranslationExampleDict +from .common import SchemaPromptSpecTranslationExampleOrDict +from .common import SchemaPromptSpecTranslationFileInputSource +from .common import SchemaPromptSpecTranslationFileInputSourceDict +from .common import SchemaPromptSpecTranslationFileInputSourceOrDict +from .common import SchemaPromptSpecTranslationGcsInputSource +from .common import SchemaPromptSpecTranslationGcsInputSourceDict +from .common import SchemaPromptSpecTranslationGcsInputSourceOrDict +from .common import SchemaPromptSpecTranslationOption +from .common import SchemaPromptSpecTranslationOptionDict +from .common import SchemaPromptSpecTranslationOptionOrDict +from .common import SchemaPromptSpecTranslationPrompt +from .common import SchemaPromptSpecTranslationPromptDict +from .common import SchemaPromptSpecTranslationPromptOrDict +from .common import SchemaPromptSpecTranslationSentenceFileInput +from .common import SchemaPromptSpecTranslationSentenceFileInputDict +from .common import SchemaPromptSpecTranslationSentenceFileInputOrDict +from .common import SchemaTablesDatasetMetadata +from .common import SchemaTablesDatasetMetadataBigQuerySource +from .common import SchemaTablesDatasetMetadataBigQuerySourceDict +from .common import SchemaTablesDatasetMetadataBigQuerySourceOrDict +from .common import SchemaTablesDatasetMetadataDict +from .common import SchemaTablesDatasetMetadataInputConfig +from .common import SchemaTablesDatasetMetadataInputConfigDict +from .common import SchemaTablesDatasetMetadataInputConfigOrDict +from .common import SchemaTablesDatasetMetadataOrDict +from .common import SchemaTextPromptDatasetMetadata +from .common import SchemaTextPromptDatasetMetadataDict +from .common import SchemaTextPromptDatasetMetadataOrDict +from .common import SecretEnvVar +from .common import SecretEnvVarDict +from .common import SecretEnvVarOrDict +from .common import SecretRef +from .common import SecretRefDict +from .common import SecretRefOrDict +from .common import Session +from .common import SessionDict +from .common import SessionEvent +from .common import SessionEventDict +from .common import SessionEventOrDict +from .common import SessionInput +from .common import SessionInputDict +from .common import SessionInputOrDict +from .common import SessionOrDict +from .common import State +from .common import Strategy +from .common import SummaryMetric +from .common import SummaryMetricDict +from .common import SummaryMetricOrDict +from .common import ToolCallValidInput +from .common import ToolCallValidInputDict +from .common import ToolCallValidInputOrDict +from .common import ToolCallValidInstance +from .common import ToolCallValidInstanceDict +from .common import ToolCallValidInstanceOrDict +from .common import ToolCallValidMetricValue +from .common import ToolCallValidMetricValueDict +from .common import ToolCallValidMetricValueOrDict +from .common import ToolCallValidResults +from .common import ToolCallValidResultsDict +from .common import ToolCallValidResultsOrDict +from .common import ToolCallValidSpec +from .common import ToolCallValidSpecDict +from .common import ToolCallValidSpecOrDict +from .common import ToolNameMatchInput +from .common import ToolNameMatchInputDict +from .common import ToolNameMatchInputOrDict +from .common import ToolNameMatchInstance +from .common import ToolNameMatchInstanceDict +from .common import ToolNameMatchInstanceOrDict +from .common import ToolNameMatchMetricValue +from .common import ToolNameMatchMetricValueDict +from .common import ToolNameMatchMetricValueOrDict +from .common import ToolNameMatchResults +from .common import ToolNameMatchResultsDict +from .common import ToolNameMatchResultsOrDict +from .common import ToolNameMatchSpec +from .common import ToolNameMatchSpecDict +from .common import ToolNameMatchSpecOrDict +from .common import ToolParameterKeyMatchInput +from .common import ToolParameterKeyMatchInputDict +from .common import ToolParameterKeyMatchInputOrDict +from .common import ToolParameterKeyMatchInstance +from .common import ToolParameterKeyMatchInstanceDict +from .common import ToolParameterKeyMatchInstanceOrDict +from .common import ToolParameterKeyMatchMetricValue +from .common import ToolParameterKeyMatchMetricValueDict +from .common import ToolParameterKeyMatchMetricValueOrDict +from .common import ToolParameterKeyMatchResults +from .common import ToolParameterKeyMatchResultsDict +from .common import ToolParameterKeyMatchResultsOrDict +from .common import ToolParameterKeyMatchSpec +from .common import ToolParameterKeyMatchSpecDict +from .common import ToolParameterKeyMatchSpecOrDict +from .common import ToolParameterKVMatchInput +from .common import ToolParameterKVMatchInputDict +from .common import ToolParameterKVMatchInputOrDict +from .common import ToolParameterKVMatchInstance +from .common import ToolParameterKVMatchInstanceDict +from .common import ToolParameterKVMatchInstanceOrDict +from .common import ToolParameterKVMatchMetricValue +from .common import ToolParameterKVMatchMetricValueDict +from .common import ToolParameterKVMatchMetricValueOrDict +from .common import ToolParameterKVMatchResults +from .common import ToolParameterKVMatchResultsDict +from .common import ToolParameterKVMatchResultsOrDict +from .common import ToolParameterKVMatchSpec +from .common import ToolParameterKVMatchSpecDict +from .common import ToolParameterKVMatchSpecOrDict +from .common import Type +from .common import UnifiedMetric +from .common import UnifiedMetricDict +from .common import UnifiedMetricOrDict +from .common import UpdateAgentEngineConfig +from .common import UpdateAgentEngineConfigDict +from .common import UpdateAgentEngineConfigOrDict +from .common import UpdateAgentEngineMemoryConfig +from .common import UpdateAgentEngineMemoryConfigDict +from .common import UpdateAgentEngineMemoryConfigOrDict +from .common import UpdateAgentEngineSessionConfig +from .common import UpdateAgentEngineSessionConfigDict +from .common import UpdateAgentEngineSessionConfigOrDict +from .common import UpdateDatasetConfig +from .common import UpdateDatasetConfigDict +from .common import UpdateDatasetConfigOrDict +from .common import VertexBaseConfig +from .common import VertexBaseConfigDict +from .common import VertexBaseConfigOrDict +from .common import WinRateStats +from .common import WinRateStatsDict +from .common import WinRateStatsOrDict +from .common import WorkerPoolSpec +from .common import WorkerPoolSpecDict +from .common import WorkerPoolSpecOrDict + +__all__ = [ + "CreateEvaluationItemConfig", + "CreateEvaluationItemConfigDict", + "CreateEvaluationItemConfigOrDict", + "PromptTemplateData", + "PromptTemplateDataDict", + "PromptTemplateDataOrDict", + "EvaluationPrompt", + "EvaluationPromptDict", + "EvaluationPromptOrDict", + "CandidateResponse", + "CandidateResponseDict", + "CandidateResponseOrDict", + "EvaluationItemRequest", + "EvaluationItemRequestDict", + "EvaluationItemRequestOrDict", + "RubricContentProperty", + "RubricContentPropertyDict", + "RubricContentPropertyOrDict", + "RubricContent", + "RubricContentDict", + "RubricContentOrDict", + "Rubric", + "RubricDict", + "RubricOrDict", + "RubricVerdict", + "RubricVerdictDict", + "RubricVerdictOrDict", + "CandidateResult", + "CandidateResultDict", + "CandidateResultOrDict", + "EvaluationItemResult", + "EvaluationItemResultDict", + "EvaluationItemResultOrDict", + "EvaluationItem", + "EvaluationItemDict", + "EvaluationItemOrDict", + "SamplingConfig", + "SamplingConfigDict", + "SamplingConfigOrDict", + "BigQueryRequestSet", + "BigQueryRequestSetDict", + "BigQueryRequestSetOrDict", + "EvaluationRunDataSource", + "EvaluationRunDataSourceDict", + "EvaluationRunDataSourceOrDict", + "PredefinedMetricSpec", + "PredefinedMetricSpecDict", + "PredefinedMetricSpecOrDict", + "RubricGenerationSpec", + "RubricGenerationSpecDict", + "RubricGenerationSpecOrDict", + "LLMBasedMetricSpec", + "LLMBasedMetricSpecDict", + "LLMBasedMetricSpecOrDict", + "UnifiedMetric", + "UnifiedMetricDict", + "UnifiedMetricOrDict", + "EvaluationRunMetric", + "EvaluationRunMetricDict", + "EvaluationRunMetricOrDict", + "EvaluationRunConfig", + "EvaluationRunConfigDict", + "EvaluationRunConfigOrDict", + "CreateEvaluationRunConfig", + "CreateEvaluationRunConfigDict", + "CreateEvaluationRunConfigOrDict", + "SummaryMetric", + "SummaryMetricDict", + "SummaryMetricOrDict", + "EvaluationRunResults", + "EvaluationRunResultsDict", + "EvaluationRunResultsOrDict", + "ResponseCandidateResult", + "ResponseCandidateResultDict", + "ResponseCandidateResultOrDict", + "EvalCaseResult", + "EvalCaseResultDict", + "EvalCaseResultOrDict", + "AggregatedMetricResult", + "AggregatedMetricResultDict", + "AggregatedMetricResultOrDict", + "ResponseCandidate", + "ResponseCandidateDict", + "ResponseCandidateOrDict", + "Event", + "EventDict", + "EventOrDict", + "Message", + "MessageDict", + "MessageOrDict", + "EvalCase", + "EvalCaseDict", + "EvalCaseOrDict", + "GcsSource", + "GcsSourceDict", + "GcsSourceOrDict", + "BigQuerySource", + "BigQuerySourceDict", + "BigQuerySourceOrDict", + "EvaluationDataset", + "EvaluationDatasetDict", + "EvaluationDatasetOrDict", + "EvaluationRunMetadata", + "EvaluationRunMetadataDict", + "EvaluationRunMetadataOrDict", + "EvaluationResult", + "EvaluationResultDict", + "EvaluationResultOrDict", + "EvaluationRun", + "EvaluationRunDict", + "EvaluationRunOrDict", + "CreateEvaluationSetConfig", + "CreateEvaluationSetConfigDict", + "CreateEvaluationSetConfigOrDict", + "EvaluationSet", + "EvaluationSetDict", + "EvaluationSetOrDict", + "BleuInstance", + "BleuInstanceDict", + "BleuInstanceOrDict", + "BleuInput", + "BleuInputDict", + "BleuInputOrDict", + "ExactMatchInstance", + "ExactMatchInstanceDict", + "ExactMatchInstanceOrDict", + "ExactMatchSpec", + "ExactMatchSpecDict", + "ExactMatchSpecOrDict", + "ExactMatchInput", + "ExactMatchInputDict", + "ExactMatchInputOrDict", + "RougeInstance", + "RougeInstanceDict", + "RougeInstanceOrDict", + "RougeInput", + "RougeInputDict", + "RougeInputOrDict", + "ContentMap", + "ContentMapDict", + "ContentMapOrDict", + "PointwiseMetricInstance", + "PointwiseMetricInstanceDict", + "PointwiseMetricInstanceOrDict", + "PointwiseMetricInput", + "PointwiseMetricInputDict", + "PointwiseMetricInputOrDict", + "PairwiseMetricInstance", + "PairwiseMetricInstanceDict", + "PairwiseMetricInstanceOrDict", + "PairwiseMetricInput", + "PairwiseMetricInputDict", + "PairwiseMetricInputOrDict", + "ToolCallValidInstance", + "ToolCallValidInstanceDict", + "ToolCallValidInstanceOrDict", + "ToolCallValidSpec", + "ToolCallValidSpecDict", + "ToolCallValidSpecOrDict", + "ToolCallValidInput", + "ToolCallValidInputDict", + "ToolCallValidInputOrDict", + "ToolNameMatchInstance", + "ToolNameMatchInstanceDict", + "ToolNameMatchInstanceOrDict", + "ToolNameMatchSpec", + "ToolNameMatchSpecDict", + "ToolNameMatchSpecOrDict", + "ToolNameMatchInput", + "ToolNameMatchInputDict", + "ToolNameMatchInputOrDict", + "ToolParameterKeyMatchInstance", + "ToolParameterKeyMatchInstanceDict", + "ToolParameterKeyMatchInstanceOrDict", + "ToolParameterKeyMatchSpec", + "ToolParameterKeyMatchSpecDict", + "ToolParameterKeyMatchSpecOrDict", + "ToolParameterKeyMatchInput", + "ToolParameterKeyMatchInputDict", + "ToolParameterKeyMatchInputOrDict", + "ToolParameterKVMatchInstance", + "ToolParameterKVMatchInstanceDict", + "ToolParameterKVMatchInstanceOrDict", + "ToolParameterKVMatchSpec", + "ToolParameterKVMatchSpecDict", + "ToolParameterKVMatchSpecOrDict", + "ToolParameterKVMatchInput", + "ToolParameterKVMatchInputDict", + "ToolParameterKVMatchInputOrDict", + "MapInstance", + "MapInstanceDict", + "MapInstanceOrDict", + "EvaluationInstance", + "EvaluationInstanceDict", + "EvaluationInstanceOrDict", + "EvaluateInstancesConfig", + "EvaluateInstancesConfigDict", + "EvaluateInstancesConfigOrDict", + "RubricBasedMetricSpec", + "RubricBasedMetricSpecDict", + "RubricBasedMetricSpecOrDict", + "RubricEnhancedContents", + "RubricEnhancedContentsDict", + "RubricEnhancedContentsOrDict", + "RubricBasedMetricInstance", + "RubricBasedMetricInstanceDict", + "RubricBasedMetricInstanceOrDict", + "RubricBasedMetricInput", + "RubricBasedMetricInputDict", + "RubricBasedMetricInputOrDict", + "Metric", + "MetricDict", + "MetricOrDict", + "MetricResult", + "MetricResultDict", + "MetricResultOrDict", + "RubricBasedMetricResult", + "RubricBasedMetricResultDict", + "RubricBasedMetricResultOrDict", + "BleuMetricValue", + "BleuMetricValueDict", + "BleuMetricValueOrDict", + "BleuResults", + "BleuResultsDict", + "BleuResultsOrDict", + "CometResult", + "CometResultDict", + "CometResultOrDict", + "ExactMatchMetricValue", + "ExactMatchMetricValueDict", + "ExactMatchMetricValueOrDict", + "ExactMatchResults", + "ExactMatchResultsDict", + "ExactMatchResultsOrDict", + "MetricxResult", + "MetricxResultDict", + "MetricxResultOrDict", + "RawOutput", + "RawOutputDict", + "RawOutputOrDict", + "CustomOutput", + "CustomOutputDict", + "CustomOutputOrDict", + "PairwiseMetricResult", + "PairwiseMetricResultDict", + "PairwiseMetricResultOrDict", + "PointwiseMetricResult", + "PointwiseMetricResultDict", + "PointwiseMetricResultOrDict", + "RougeMetricValue", + "RougeMetricValueDict", + "RougeMetricValueOrDict", + "RougeResults", + "RougeResultsDict", + "RougeResultsOrDict", + "ToolCallValidMetricValue", + "ToolCallValidMetricValueDict", + "ToolCallValidMetricValueOrDict", + "ToolCallValidResults", + "ToolCallValidResultsDict", + "ToolCallValidResultsOrDict", + "ToolNameMatchMetricValue", + "ToolNameMatchMetricValueDict", + "ToolNameMatchMetricValueOrDict", + "ToolNameMatchResults", + "ToolNameMatchResultsDict", + "ToolNameMatchResultsOrDict", + "ToolParameterKeyMatchMetricValue", + "ToolParameterKeyMatchMetricValueDict", + "ToolParameterKeyMatchMetricValueOrDict", + "ToolParameterKeyMatchResults", + "ToolParameterKeyMatchResultsDict", + "ToolParameterKeyMatchResultsOrDict", + "ToolParameterKVMatchMetricValue", + "ToolParameterKVMatchMetricValueDict", + "ToolParameterKVMatchMetricValueOrDict", + "ToolParameterKVMatchResults", + "ToolParameterKVMatchResultsDict", + "ToolParameterKVMatchResultsOrDict", + "EvaluateInstancesResponse", + "EvaluateInstancesResponseDict", + "EvaluateInstancesResponseOrDict", + "RubricGenerationConfig", + "RubricGenerationConfigDict", + "RubricGenerationConfigOrDict", + "GenerateInstanceRubricsResponse", + "GenerateInstanceRubricsResponseDict", + "GenerateInstanceRubricsResponseOrDict", + "GetEvaluationRunConfig", + "GetEvaluationRunConfigDict", + "GetEvaluationRunConfigOrDict", + "GetEvaluationSetConfig", + "GetEvaluationSetConfigDict", + "GetEvaluationSetConfigOrDict", + "GetEvaluationItemConfig", + "GetEvaluationItemConfigDict", + "GetEvaluationItemConfigOrDict", + "OptimizeConfig", + "OptimizeConfigDict", + "OptimizeConfigOrDict", + "OptimizeResponseEndpoint", + "OptimizeResponseEndpointDict", + "OptimizeResponseEndpointOrDict", + "DnsPeeringConfig", + "DnsPeeringConfigDict", + "DnsPeeringConfigOrDict", + "PscInterfaceConfig", + "PscInterfaceConfigDict", + "PscInterfaceConfigOrDict", + "Scheduling", + "SchedulingDict", + "SchedulingOrDict", + "EnvVar", + "EnvVarDict", + "EnvVarOrDict", + "ContainerSpec", + "ContainerSpecDict", + "ContainerSpecOrDict", + "DiskSpec", + "DiskSpecDict", + "DiskSpecOrDict", + "ReservationAffinity", + "ReservationAffinityDict", + "ReservationAffinityOrDict", + "MachineSpec", + "MachineSpecDict", + "MachineSpecOrDict", + "NfsMount", + "NfsMountDict", + "NfsMountOrDict", + "PythonPackageSpec", + "PythonPackageSpecDict", + "PythonPackageSpecOrDict", + "WorkerPoolSpec", + "WorkerPoolSpecDict", + "WorkerPoolSpecOrDict", + "CustomJobSpec", + "CustomJobSpecDict", + "CustomJobSpecOrDict", + "CustomJob", + "CustomJobDict", + "CustomJobOrDict", + "VertexBaseConfig", + "VertexBaseConfigDict", + "VertexBaseConfigOrDict", + "SecretRef", + "SecretRefDict", + "SecretRefOrDict", + "SecretEnvVar", + "SecretEnvVarDict", + "SecretEnvVarOrDict", + "ReasoningEngineSpecDeploymentSpec", + "ReasoningEngineSpecDeploymentSpecDict", + "ReasoningEngineSpecDeploymentSpecOrDict", + "ReasoningEngineSpecPackageSpec", + "ReasoningEngineSpecPackageSpecDict", + "ReasoningEngineSpecPackageSpecOrDict", + "ReasoningEngineSpec", + "ReasoningEngineSpecDict", + "ReasoningEngineSpecOrDict", + "ReasoningEngineContextSpecMemoryBankConfigGenerationConfig", + "ReasoningEngineContextSpecMemoryBankConfigGenerationConfigDict", + "ReasoningEngineContextSpecMemoryBankConfigGenerationConfigOrDict", + "ReasoningEngineContextSpecMemoryBankConfigSimilaritySearchConfig", + "ReasoningEngineContextSpecMemoryBankConfigSimilaritySearchConfigDict", + "ReasoningEngineContextSpecMemoryBankConfigSimilaritySearchConfigOrDict", + "MemoryBankCustomizationConfigMemoryTopicCustomMemoryTopic", + "MemoryBankCustomizationConfigMemoryTopicCustomMemoryTopicDict", + "MemoryBankCustomizationConfigMemoryTopicCustomMemoryTopicOrDict", + "MemoryBankCustomizationConfigMemoryTopicManagedMemoryTopic", + "MemoryBankCustomizationConfigMemoryTopicManagedMemoryTopicDict", + "MemoryBankCustomizationConfigMemoryTopicManagedMemoryTopicOrDict", + "MemoryBankCustomizationConfigMemoryTopic", + "MemoryBankCustomizationConfigMemoryTopicDict", + "MemoryBankCustomizationConfigMemoryTopicOrDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceEvent", + "MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceEventDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceEventOrDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSource", + "MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleConversationSourceOrDict", + "MemoryTopicId", + "MemoryTopicIdDict", + "MemoryTopicIdOrDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleGeneratedMemory", + "MemoryBankCustomizationConfigGenerateMemoriesExampleGeneratedMemoryDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleGeneratedMemoryOrDict", + "MemoryBankCustomizationConfigGenerateMemoriesExample", + "MemoryBankCustomizationConfigGenerateMemoriesExampleDict", + "MemoryBankCustomizationConfigGenerateMemoriesExampleOrDict", + "MemoryBankCustomizationConfig", + "MemoryBankCustomizationConfigDict", + "MemoryBankCustomizationConfigOrDict", + "ReasoningEngineContextSpecMemoryBankConfigTtlConfigGranularTtlConfig", + "ReasoningEngineContextSpecMemoryBankConfigTtlConfigGranularTtlConfigDict", + "ReasoningEngineContextSpecMemoryBankConfigTtlConfigGranularTtlConfigOrDict", + "ReasoningEngineContextSpecMemoryBankConfigTtlConfig", + "ReasoningEngineContextSpecMemoryBankConfigTtlConfigDict", + "ReasoningEngineContextSpecMemoryBankConfigTtlConfigOrDict", + "ReasoningEngineContextSpecMemoryBankConfig", + "ReasoningEngineContextSpecMemoryBankConfigDict", + "ReasoningEngineContextSpecMemoryBankConfigOrDict", + "ReasoningEngineContextSpec", + "ReasoningEngineContextSpecDict", + "ReasoningEngineContextSpecOrDict", + "CreateAgentEngineConfig", + "CreateAgentEngineConfigDict", + "CreateAgentEngineConfigOrDict", + "ReasoningEngine", + "ReasoningEngineDict", + "ReasoningEngineOrDict", + "AgentEngineOperation", + "AgentEngineOperationDict", + "AgentEngineOperationOrDict", + "DeleteAgentEngineConfig", + "DeleteAgentEngineConfigDict", + "DeleteAgentEngineConfigOrDict", + "DeleteAgentEngineOperation", + "DeleteAgentEngineOperationDict", + "DeleteAgentEngineOperationOrDict", + "GetAgentEngineConfig", + "GetAgentEngineConfigDict", + "GetAgentEngineConfigOrDict", + "ListAgentEngineConfig", + "ListAgentEngineConfigDict", + "ListAgentEngineConfigOrDict", + "ListReasoningEnginesResponse", + "ListReasoningEnginesResponseDict", + "ListReasoningEnginesResponseOrDict", + "GetAgentEngineOperationConfig", + "GetAgentEngineOperationConfigDict", + "GetAgentEngineOperationConfigOrDict", + "QueryAgentEngineConfig", + "QueryAgentEngineConfigDict", + "QueryAgentEngineConfigOrDict", + "QueryReasoningEngineResponse", + "QueryReasoningEngineResponseDict", + "QueryReasoningEngineResponseOrDict", + "UpdateAgentEngineConfig", + "UpdateAgentEngineConfigDict", + "UpdateAgentEngineConfigOrDict", + "AgentEngineMemoryConfig", + "AgentEngineMemoryConfigDict", + "AgentEngineMemoryConfigOrDict", + "Memory", + "MemoryDict", + "MemoryOrDict", + "AgentEngineMemoryOperation", + "AgentEngineMemoryOperationDict", + "AgentEngineMemoryOperationOrDict", + "DeleteAgentEngineMemoryConfig", + "DeleteAgentEngineMemoryConfigDict", + "DeleteAgentEngineMemoryConfigOrDict", + "DeleteAgentEngineMemoryOperation", + "DeleteAgentEngineMemoryOperationDict", + "DeleteAgentEngineMemoryOperationOrDict", + "GenerateMemoriesRequestVertexSessionSource", + "GenerateMemoriesRequestVertexSessionSourceDict", + "GenerateMemoriesRequestVertexSessionSourceOrDict", + "GenerateMemoriesRequestDirectContentsSourceEvent", + "GenerateMemoriesRequestDirectContentsSourceEventDict", + "GenerateMemoriesRequestDirectContentsSourceEventOrDict", + "GenerateMemoriesRequestDirectContentsSource", + "GenerateMemoriesRequestDirectContentsSourceDict", + "GenerateMemoriesRequestDirectContentsSourceOrDict", + "GenerateMemoriesRequestDirectMemoriesSourceDirectMemory", + "GenerateMemoriesRequestDirectMemoriesSourceDirectMemoryDict", + "GenerateMemoriesRequestDirectMemoriesSourceDirectMemoryOrDict", + "GenerateMemoriesRequestDirectMemoriesSource", + "GenerateMemoriesRequestDirectMemoriesSourceDict", + "GenerateMemoriesRequestDirectMemoriesSourceOrDict", + "GenerateAgentEngineMemoriesConfig", + "GenerateAgentEngineMemoriesConfigDict", + "GenerateAgentEngineMemoriesConfigOrDict", + "GenerateMemoriesResponseGeneratedMemory", + "GenerateMemoriesResponseGeneratedMemoryDict", + "GenerateMemoriesResponseGeneratedMemoryOrDict", + "GenerateMemoriesResponse", + "GenerateMemoriesResponseDict", + "GenerateMemoriesResponseOrDict", + "AgentEngineGenerateMemoriesOperation", + "AgentEngineGenerateMemoriesOperationDict", + "AgentEngineGenerateMemoriesOperationOrDict", + "GetAgentEngineMemoryConfig", + "GetAgentEngineMemoryConfigDict", + "GetAgentEngineMemoryConfigOrDict", + "ListAgentEngineMemoryConfig", + "ListAgentEngineMemoryConfigDict", + "ListAgentEngineMemoryConfigOrDict", + "ListReasoningEnginesMemoriesResponse", + "ListReasoningEnginesMemoriesResponseDict", + "ListReasoningEnginesMemoriesResponseOrDict", + "RetrieveMemoriesRequestSimilaritySearchParams", + "RetrieveMemoriesRequestSimilaritySearchParamsDict", + "RetrieveMemoriesRequestSimilaritySearchParamsOrDict", + "RetrieveMemoriesRequestSimpleRetrievalParams", + "RetrieveMemoriesRequestSimpleRetrievalParamsDict", + "RetrieveMemoriesRequestSimpleRetrievalParamsOrDict", + "RetrieveAgentEngineMemoriesConfig", + "RetrieveAgentEngineMemoriesConfigDict", + "RetrieveAgentEngineMemoriesConfigOrDict", + "RetrieveMemoriesResponseRetrievedMemory", + "RetrieveMemoriesResponseRetrievedMemoryDict", + "RetrieveMemoriesResponseRetrievedMemoryOrDict", + "RetrieveMemoriesResponse", + "RetrieveMemoriesResponseDict", + "RetrieveMemoriesResponseOrDict", + "RollbackAgentEngineMemoryConfig", + "RollbackAgentEngineMemoryConfigDict", + "RollbackAgentEngineMemoryConfigOrDict", + "AgentEngineRollbackMemoryOperation", + "AgentEngineRollbackMemoryOperationDict", + "AgentEngineRollbackMemoryOperationOrDict", + "UpdateAgentEngineMemoryConfig", + "UpdateAgentEngineMemoryConfigDict", + "UpdateAgentEngineMemoryConfigOrDict", + "GetAgentEngineMemoryRevisionConfig", + "GetAgentEngineMemoryRevisionConfigDict", + "GetAgentEngineMemoryRevisionConfigOrDict", + "MemoryRevision", + "MemoryRevisionDict", + "MemoryRevisionOrDict", + "ListAgentEngineMemoryRevisionsConfig", + "ListAgentEngineMemoryRevisionsConfigDict", + "ListAgentEngineMemoryRevisionsConfigOrDict", + "ListAgentEngineMemoryRevisionsResponse", + "ListAgentEngineMemoryRevisionsResponseDict", + "ListAgentEngineMemoryRevisionsResponseOrDict", + "SandboxEnvironmentSpecCodeExecutionEnvironment", + "SandboxEnvironmentSpecCodeExecutionEnvironmentDict", + "SandboxEnvironmentSpecCodeExecutionEnvironmentOrDict", + "SandboxEnvironmentSpecComputerUseEnvironment", + "SandboxEnvironmentSpecComputerUseEnvironmentDict", + "SandboxEnvironmentSpecComputerUseEnvironmentOrDict", + "SandboxEnvironmentSpec", + "SandboxEnvironmentSpecDict", + "SandboxEnvironmentSpecOrDict", + "CreateAgentEngineSandboxConfig", + "CreateAgentEngineSandboxConfigDict", + "CreateAgentEngineSandboxConfigOrDict", + "SandboxEnvironmentConnectionInfo", + "SandboxEnvironmentConnectionInfoDict", + "SandboxEnvironmentConnectionInfoOrDict", + "SandboxEnvironment", + "SandboxEnvironmentDict", + "SandboxEnvironmentOrDict", + "AgentEngineSandboxOperation", + "AgentEngineSandboxOperationDict", + "AgentEngineSandboxOperationOrDict", + "DeleteAgentEngineSandboxConfig", + "DeleteAgentEngineSandboxConfigDict", + "DeleteAgentEngineSandboxConfigOrDict", + "DeleteAgentEngineSandboxOperation", + "DeleteAgentEngineSandboxOperationDict", + "DeleteAgentEngineSandboxOperationOrDict", + "Metadata", + "MetadataDict", + "MetadataOrDict", + "Chunk", + "ChunkDict", + "ChunkOrDict", + "ExecuteCodeAgentEngineSandboxConfig", + "ExecuteCodeAgentEngineSandboxConfigDict", + "ExecuteCodeAgentEngineSandboxConfigOrDict", + "ExecuteSandboxEnvironmentResponse", + "ExecuteSandboxEnvironmentResponseDict", + "ExecuteSandboxEnvironmentResponseOrDict", + "GetAgentEngineSandboxConfig", + "GetAgentEngineSandboxConfigDict", + "GetAgentEngineSandboxConfigOrDict", + "ListAgentEngineSandboxesConfig", + "ListAgentEngineSandboxesConfigDict", + "ListAgentEngineSandboxesConfigOrDict", + "ListAgentEngineSandboxesResponse", + "ListAgentEngineSandboxesResponseDict", + "ListAgentEngineSandboxesResponseOrDict", + "CreateAgentEngineSessionConfig", + "CreateAgentEngineSessionConfigDict", + "CreateAgentEngineSessionConfigOrDict", + "Session", + "SessionDict", + "SessionOrDict", + "AgentEngineSessionOperation", + "AgentEngineSessionOperationDict", + "AgentEngineSessionOperationOrDict", + "DeleteAgentEngineSessionConfig", + "DeleteAgentEngineSessionConfigDict", + "DeleteAgentEngineSessionConfigOrDict", + "DeleteAgentEngineSessionOperation", + "DeleteAgentEngineSessionOperationDict", + "DeleteAgentEngineSessionOperationOrDict", + "GetAgentEngineSessionConfig", + "GetAgentEngineSessionConfigDict", + "GetAgentEngineSessionConfigOrDict", + "ListAgentEngineSessionsConfig", + "ListAgentEngineSessionsConfigDict", + "ListAgentEngineSessionsConfigOrDict", + "ListReasoningEnginesSessionsResponse", + "ListReasoningEnginesSessionsResponseDict", + "ListReasoningEnginesSessionsResponseOrDict", + "UpdateAgentEngineSessionConfig", + "UpdateAgentEngineSessionConfigDict", + "UpdateAgentEngineSessionConfigOrDict", + "EventActions", + "EventActionsDict", + "EventActionsOrDict", + "EventMetadata", + "EventMetadataDict", + "EventMetadataOrDict", + "AppendAgentEngineSessionEventConfig", + "AppendAgentEngineSessionEventConfigDict", + "AppendAgentEngineSessionEventConfigOrDict", + "AppendAgentEngineSessionEventResponse", + "AppendAgentEngineSessionEventResponseDict", + "AppendAgentEngineSessionEventResponseOrDict", + "ListAgentEngineSessionEventsConfig", + "ListAgentEngineSessionEventsConfigDict", + "ListAgentEngineSessionEventsConfigOrDict", + "SessionEvent", + "SessionEventDict", + "SessionEventOrDict", + "ListAgentEngineSessionEventsResponse", + "ListAgentEngineSessionEventsResponseDict", + "ListAgentEngineSessionEventsResponseOrDict", + "CreateMultimodalDatasetConfig", + "CreateMultimodalDatasetConfigDict", + "CreateMultimodalDatasetConfigOrDict", + "SchemaTablesDatasetMetadataBigQuerySource", + "SchemaTablesDatasetMetadataBigQuerySourceDict", + "SchemaTablesDatasetMetadataBigQuerySourceOrDict", + "SchemaTablesDatasetMetadataInputConfig", + "SchemaTablesDatasetMetadataInputConfigDict", + "SchemaTablesDatasetMetadataInputConfigOrDict", + "SchemaTablesDatasetMetadata", + "SchemaTablesDatasetMetadataDict", + "SchemaTablesDatasetMetadataOrDict", + "MultimodalDataset", + "MultimodalDatasetDict", + "MultimodalDatasetOrDict", + "MultimodalDatasetOperation", + "MultimodalDatasetOperationDict", + "MultimodalDatasetOperationOrDict", + "GetMultimodalDatasetOperationConfig", + "GetMultimodalDatasetOperationConfigDict", + "GetMultimodalDatasetOperationConfigOrDict", + "CreateDatasetConfig", + "CreateDatasetConfigDict", + "CreateDatasetConfigOrDict", + "SchemaPredictParamsGroundingConfigSourceEntry", + "SchemaPredictParamsGroundingConfigSourceEntryDict", + "SchemaPredictParamsGroundingConfigSourceEntryOrDict", + "SchemaPredictParamsGroundingConfig", + "SchemaPredictParamsGroundingConfigDict", + "SchemaPredictParamsGroundingConfigOrDict", + "SchemaPromptInstancePromptExecution", + "SchemaPromptInstancePromptExecutionDict", + "SchemaPromptInstancePromptExecutionOrDict", + "SchemaPromptSpecPromptMessage", + "SchemaPromptSpecPromptMessageDict", + "SchemaPromptSpecPromptMessageOrDict", + "SchemaPromptSpecMultimodalPrompt", + "SchemaPromptSpecMultimodalPromptDict", + "SchemaPromptSpecMultimodalPromptOrDict", + "SchemaPromptSpecPartList", + "SchemaPromptSpecPartListDict", + "SchemaPromptSpecPartListOrDict", + "SchemaPromptSpecStructuredPrompt", + "SchemaPromptSpecStructuredPromptDict", + "SchemaPromptSpecStructuredPromptOrDict", + "SchemaPromptSpecReferenceSentencePair", + "SchemaPromptSpecReferenceSentencePairDict", + "SchemaPromptSpecReferenceSentencePairOrDict", + "SchemaPromptSpecReferenceSentencePairList", + "SchemaPromptSpecReferenceSentencePairListDict", + "SchemaPromptSpecReferenceSentencePairListOrDict", + "SchemaPromptSpecTranslationFileInputSource", + "SchemaPromptSpecTranslationFileInputSourceDict", + "SchemaPromptSpecTranslationFileInputSourceOrDict", + "SchemaPromptSpecTranslationGcsInputSource", + "SchemaPromptSpecTranslationGcsInputSourceDict", + "SchemaPromptSpecTranslationGcsInputSourceOrDict", + "SchemaPromptSpecTranslationSentenceFileInput", + "SchemaPromptSpecTranslationSentenceFileInputDict", + "SchemaPromptSpecTranslationSentenceFileInputOrDict", + "SchemaPromptSpecTranslationExample", + "SchemaPromptSpecTranslationExampleDict", + "SchemaPromptSpecTranslationExampleOrDict", + "SchemaPromptSpecTranslationOption", + "SchemaPromptSpecTranslationOptionDict", + "SchemaPromptSpecTranslationOptionOrDict", + "SchemaPromptSpecTranslationPrompt", + "SchemaPromptSpecTranslationPromptDict", + "SchemaPromptSpecTranslationPromptOrDict", + "SchemaPromptApiSchema", + "SchemaPromptApiSchemaDict", + "SchemaPromptApiSchemaOrDict", + "SchemaTextPromptDatasetMetadata", + "SchemaTextPromptDatasetMetadataDict", + "SchemaTextPromptDatasetMetadataOrDict", + "DatasetOperation", + "DatasetOperationDict", + "DatasetOperationOrDict", + "CreateDatasetVersionConfig", + "CreateDatasetVersionConfigDict", + "CreateDatasetVersionConfigOrDict", + "SavedQuery", + "SavedQueryDict", + "SavedQueryOrDict", + "Dataset", + "DatasetDict", + "DatasetOrDict", + "DatasetVersion", + "DatasetVersionDict", + "DatasetVersionOrDict", + "GetDatasetOperationConfig", + "GetDatasetOperationConfigDict", + "GetDatasetOperationConfigOrDict", + "ListPromptsConfig", + "ListPromptsConfigDict", + "ListPromptsConfigOrDict", + "ListDatasetsResponse", + "ListDatasetsResponseDict", + "ListDatasetsResponseOrDict", + "ListDatasetVersionsResponse", + "ListDatasetVersionsResponseDict", + "ListDatasetVersionsResponseOrDict", + "DeletePromptConfig", + "DeletePromptConfigDict", + "DeletePromptConfigOrDict", + "DeletePromptOperation", + "DeletePromptOperationDict", + "DeletePromptOperationOrDict", + "DeletePromptVersionOperation", + "DeletePromptVersionOperationDict", + "DeletePromptVersionOperationOrDict", + "RestoreVersionConfig", + "RestoreVersionConfigDict", + "RestoreVersionConfigOrDict", + "RestoreVersionOperation", + "RestoreVersionOperationDict", + "RestoreVersionOperationOrDict", + "UpdateDatasetConfig", + "UpdateDatasetConfigDict", + "UpdateDatasetConfigOrDict", + "PromptOptimizerVAPOConfig", + "PromptOptimizerVAPOConfigDict", + "PromptOptimizerVAPOConfigOrDict", + "ApplicableGuideline", + "ApplicableGuidelineDict", + "ApplicableGuidelineOrDict", + "ParsedResponse", + "ParsedResponseDict", + "ParsedResponseOrDict", + "OptimizeResponse", + "OptimizeResponseDict", + "OptimizeResponseOrDict", + "PromptTemplate", + "PromptTemplateDict", + "PromptTemplateOrDict", + "EvalRunInferenceConfig", + "EvalRunInferenceConfigDict", + "EvalRunInferenceConfigOrDict", + "ContentMapContents", + "ContentMapContentsDict", + "ContentMapContentsOrDict", + "EvalCaseMetricResult", + "EvalCaseMetricResultDict", + "EvalCaseMetricResultOrDict", + "EvaluationRunAgentConfig", + "EvaluationRunAgentConfigDict", + "EvaluationRunAgentConfigOrDict", + "EvaluationRunInferenceConfig", + "EvaluationRunInferenceConfigDict", + "EvaluationRunInferenceConfigOrDict", + "SessionInput", + "SessionInputDict", + "SessionInputOrDict", + "WinRateStats", + "WinRateStatsDict", + "WinRateStatsOrDict", + "EvaluateMethodConfig", + "EvaluateMethodConfigDict", + "EvaluateMethodConfigOrDict", + "EvaluateDatasetConfig", + "EvaluateDatasetConfigDict", + "EvaluateDatasetConfigOrDict", + "EvaluateDatasetRequestParameters", + "EvaluateDatasetRequestParametersDict", + "EvaluateDatasetRequestParametersOrDict", + "EvaluateDatasetOperation", + "EvaluateDatasetOperationDict", + "EvaluateDatasetOperationOrDict", + "ObservabilityEvalCase", + "ObservabilityEvalCaseDict", + "ObservabilityEvalCaseOrDict", + "RubricGroup", + "RubricGroupDict", + "RubricGroupOrDict", + "AgentEngine", + "AgentEngineDict", + "AgentEngineOrDict", + "AgentEngineConfig", + "AgentEngineConfigDict", + "AgentEngineConfigOrDict", + "Prompt", + "PromptDict", + "PromptOrDict", + "SchemaPromptInstanceVariableValue", + "SchemaPromptInstanceVariableValueDict", + "SchemaPromptInstanceVariableValueOrDict", + "CreatePromptConfig", + "CreatePromptConfigDict", + "CreatePromptConfigOrDict", + "CreatePromptVersionConfig", + "CreatePromptVersionConfigDict", + "CreatePromptVersionConfigOrDict", + "GetPromptConfig", + "GetPromptConfigDict", + "GetPromptConfigOrDict", + "PromptRef", + "PromptRefDict", + "PromptRefOrDict", + "PromptVersionRef", + "PromptVersionRefDict", + "PromptVersionRefOrDict", + "PairwiseChoice", + "Strategy", + "AcceleratorType", + "Type", + "JobState", + "AgentServerMode", + "ManagedTopicEnum", + "Language", + "MachineConfig", + "State", + "EvaluationItemType", + "SamplingMethod", + "RubricContentType", + "EvaluationRunState", + "Importance", + "GenerateMemoriesResponseGeneratedMemoryAction", + "PromptData", + "PromptDataDict", + "PromptDataOrDict", + "LLMMetric", + "RubricMetric", + "MetricPromptBuilder", + "_CreateEvaluationItemParameters", + "_CreateEvaluationRunParameters", + "_CreateEvaluationSetParameters", + "_EvaluateInstancesRequestParameters", + "_GenerateInstanceRubricsRequest", + "_GetEvaluationRunParameters", + "_GetEvaluationSetParameters", + "_GetEvaluationItemParameters", + "_OptimizeRequestParameters", + "_CustomJobParameters", + "_GetCustomJobParameters", + "_CreateAgentEngineRequestParameters", + "_DeleteAgentEngineRequestParameters", + "_GetAgentEngineRequestParameters", + "_ListAgentEngineRequestParameters", + "_GetAgentEngineOperationParameters", + "_QueryAgentEngineRequestParameters", + "_UpdateAgentEngineRequestParameters", + "_CreateAgentEngineMemoryRequestParameters", + "_DeleteAgentEngineMemoryRequestParameters", + "_GenerateAgentEngineMemoriesRequestParameters", + "_GetAgentEngineMemoryRequestParameters", + "_ListAgentEngineMemoryRequestParameters", + "_GetAgentEngineMemoryOperationParameters", + "_GetAgentEngineGenerateMemoriesOperationParameters", + "_RetrieveAgentEngineMemoriesRequestParameters", + "_RollbackAgentEngineMemoryRequestParameters", + "_UpdateAgentEngineMemoryRequestParameters", + "_GetAgentEngineMemoryRevisionRequestParameters", + "_ListAgentEngineMemoryRevisionsRequestParameters", + "_CreateAgentEngineSandboxRequestParameters", + "_DeleteAgentEngineSandboxRequestParameters", + "_ExecuteCodeAgentEngineSandboxRequestParameters", + "_GetAgentEngineSandboxRequestParameters", + "_ListAgentEngineSandboxesRequestParameters", + "_GetAgentEngineSandboxOperationParameters", + "_CreateAgentEngineSessionRequestParameters", + "_DeleteAgentEngineSessionRequestParameters", + "_GetAgentEngineSessionRequestParameters", + "_ListAgentEngineSessionsRequestParameters", + "_GetAgentEngineSessionOperationParameters", + "_UpdateAgentEngineSessionRequestParameters", + "_AppendAgentEngineSessionEventRequestParameters", + "_ListAgentEngineSessionEventsRequestParameters", + "_CreateMultimodalDatasetParameters", + "_GetMultimodalDatasetOperationParameters", + "_CreateDatasetParameters", + "_CreateDatasetVersionParameters", + "_GetDatasetParameters", + "_GetDatasetVersionParameters", + "_GetDatasetOperationParameters", + "_ListDatasetsRequestParameters", + "_ListDatasetVersionsRequestParameters", + "_DeleteDatasetRequestParameters", + "_DeletePromptVersionRequestParameters", + "_RestoreVersionRequestParameters", + "_UpdateDatasetParameters", + "evals", + "agent_engines", +] diff --git a/vertexai/_genai/types/agent_engines.py b/vertexai/_genai/types/agent_engines.py new file mode 100644 index 0000000000..c47a08a11e --- /dev/null +++ b/vertexai/_genai/types/agent_engines.py @@ -0,0 +1,16 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Code generated by the Google Gen AI SDK generator DO NOT EDIT. diff --git a/vertexai/_genai/types.py b/vertexai/_genai/types/common.py similarity index 98% rename from vertexai/_genai/types.py rename to vertexai/_genai/types/common.py index e22d424c93..a27979b07d 100644 --- a/vertexai/_genai/types.py +++ b/vertexai/_genai/types/common.py @@ -45,15 +45,14 @@ model_validator, ) from typing_extensions import TypedDict - -logger = logging.getLogger("vertexai_genai.types") +from . import evals as evals_types __all__ = ["PrebuiltMetric", "RubricMetric"] # noqa: F822 def __getattr__(name: str) -> typing.Any: if name == "PrebuiltMetric" or name == "RubricMetric": - module = importlib.import_module("._evals_utils", __package__) + module = importlib.import_module(".._evals_metric_loaders", __package__) prebuilt_metric_obj = getattr(module, name) globals()[name] = prebuilt_metric_obj return prebuilt_metric_obj @@ -1483,51 +1482,6 @@ class EventDict(TypedDict, total=False): EventOrDict = Union[Event, EventDict] -class AgentInfo(_common.BaseModel): - """The agent info of an agent, used for agent eval.""" - - agent: Optional[str] = Field( - default=None, - description="""The agent engine used to run agent. Agent engine resource name in str type, with format - `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.""", - ) - name: Optional[str] = Field( - default=None, description="""Agent name, used as an identifier.""" - ) - instruction: Optional[str] = Field( - default=None, description="""Agent developer instruction.""" - ) - description: Optional[str] = Field( - default=None, description="""Agent description.""" - ) - tool_declarations: Optional[genai_types.ToolListUnion] = Field( - default=None, description="""List of tools used by the Agent.""" - ) - - -class AgentInfoDict(TypedDict, total=False): - """The agent info of an agent, used for agent eval.""" - - agent: Optional[str] - """The agent engine used to run agent. Agent engine resource name in str type, with format - `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.""" - - name: Optional[str] - """Agent name, used as an identifier.""" - - instruction: Optional[str] - """Agent developer instruction.""" - - description: Optional[str] - """Agent description.""" - - tool_declarations: Optional[genai_types.ToolListUnionDict] - """List of tools used by the Agent.""" - - -AgentInfoOrDict = Union[AgentInfo, AgentInfoDict] - - class Message(_common.BaseModel): """Represents a single message turn in a conversation.""" @@ -1597,7 +1551,7 @@ class EvalCase(_common.BaseModel): default=None, description="""This field is experimental and may change in future versions. Intermediate events of a single turn in an agent run or intermediate events of the last turn for multi-turn an agent run.""", ) - agent_info: Optional[AgentInfo] = Field( + agent_info: Optional[evals_types.AgentInfo] = Field( default=None, description="""This field is experimental and may change in future versions. The agent info of the agent under evaluation. This can be extended for multi-agent evaluation.""", ) @@ -1632,7 +1586,7 @@ class EvalCaseDict(TypedDict, total=False): intermediate_events: Optional[list[EventDict]] """This field is experimental and may change in future versions. Intermediate events of a single turn in an agent run or intermediate events of the last turn for multi-turn an agent run.""" - agent_info: Optional[AgentInfoDict] + agent_info: Optional[evals_types.AgentInfo] """This field is experimental and may change in future versions. The agent info of the agent under evaluation. This can be extended for multi-agent evaluation.""" @@ -1723,7 +1677,7 @@ def load_from_observability_eval_cases( """Fetches GenAI Observability data from GCS and parses into a DataFrame.""" try: import pandas as pd - from . import _evals_utils + from .. import _gcs_utils formats = [] requests = [] @@ -1731,7 +1685,7 @@ def load_from_observability_eval_cases( system_instructions = [] for case in cases: - gcs_utils = _evals_utils.GcsUtils( + gcs_utils = _gcs_utils.GcsUtils( case.api_client._api_client if case.api_client else None ) @@ -1770,7 +1724,7 @@ def load_from_observability_eval_cases( def show(self) -> None: """Shows the evaluation dataset.""" - from . import _evals_visualization + from .. import _evals_visualization _evals_visualization.display_evaluation_dataset(self) @@ -1858,7 +1812,7 @@ class EvaluationResult(_common.BaseModel): metadata: Optional[EvaluationRunMetadata] = Field( default=None, description="""Metadata for the evaluation run.""" ) - agent_info: Optional[AgentInfo] = Field( + agent_info: Optional[evals_types.AgentInfo] = Field( default=None, description="""This field is experimental and may change in future versions. The agent info of the agent under evaluation. This can be extended for multi-agent evaluation.""", ) @@ -1870,7 +1824,7 @@ def show(self, candidate_names: Optional[List[str]] = None) -> None: candidate_names: list of names for the evaluated candidates, used in comparison reports. """ - from . import _evals_visualization + from .. import _evals_visualization _evals_visualization.display_evaluation_result(self, candidate_names) @@ -1893,7 +1847,7 @@ class EvaluationResultDict(TypedDict, total=False): metadata: Optional[EvaluationRunMetadataDict] """Metadata for the evaluation run.""" - agent_info: Optional[AgentInfoDict] + agent_info: Optional[evals_types.AgentInfo] """This field is experimental and may change in future versions. The agent info of the agent under evaluation. This can be extended for multi-agent evaluation.""" @@ -1958,7 +1912,7 @@ def _from_response( def show(self) -> None: """Shows the evaluation result.""" - from . import _evals_visualization + from .. import _evals_visualization logger.warning(f"Evaluation Run state: {self.state}.") if self.error: @@ -2668,50 +2622,10 @@ class ToolParameterKVMatchInputDict(TypedDict, total=False): ] -class InstanceDataContents(_common.BaseModel): - """List of standard Content messages from Gemini API.""" - - contents: Optional[list[genai_types.Content]] = Field( - default=None, description="""Repeated contents.""" - ) - - -class InstanceDataContentsDict(TypedDict, total=False): - """List of standard Content messages from Gemini API.""" - - contents: Optional[list[genai_types.ContentDict]] - """Repeated contents.""" - - -InstanceDataContentsOrDict = Union[InstanceDataContents, InstanceDataContentsDict] - - -class InstanceData(_common.BaseModel): - """Instance data used to populate placeholders in a metric prompt template.""" - - text: Optional[str] = Field(default=None, description="""Text data.""") - contents: Optional[InstanceDataContents] = Field( - default=None, description="""List of Gemini content data.""" - ) - - -class InstanceDataDict(TypedDict, total=False): - """Instance data used to populate placeholders in a metric prompt template.""" - - text: Optional[str] - """Text data.""" - - contents: Optional[InstanceDataContentsDict] - """List of Gemini content data.""" - - -InstanceDataOrDict = Union[InstanceData, InstanceDataDict] - - class MapInstance(_common.BaseModel): """Instance data specified as a map.""" - map_instance: Optional[dict[str, InstanceData]] = Field( + map_instance: Optional[dict[str, evals_types.InstanceData]] = Field( default=None, description="""Map of instance data.""" ) @@ -2719,120 +2633,25 @@ class MapInstance(_common.BaseModel): class MapInstanceDict(TypedDict, total=False): """Instance data specified as a map.""" - map_instance: Optional[dict[str, InstanceDataDict]] + map_instance: Optional[dict[str, evals_types.InstanceData]] """Map of instance data.""" MapInstanceOrDict = Union[MapInstance, MapInstanceDict] -class Tools(_common.BaseModel): - """Represents a list of tools for an agent.""" - - tool: Optional[list[genai_types.Tool]] = Field( - default=None, - description="""List of tools: each tool can have multiple function declarations.""", - ) - - -class ToolsDict(TypedDict, total=False): - """Represents a list of tools for an agent.""" - - tool: Optional[list[genai_types.ToolDict]] - """List of tools: each tool can have multiple function declarations.""" - - -ToolsOrDict = Union[Tools, ToolsDict] - - -class AgentConfig(_common.BaseModel): - """Configuration for an Agent.""" - - tools_text: Optional[str] = Field( - default=None, - description="""A JSON string containing a list of tools available to an agent.""", - ) - tools: Optional[Tools] = Field(default=None, description="""List of tools.""") - developer_instruction: Optional[InstanceData] = Field( - default=None, - description="""A field containing instructions from the developer for the agent.""", - ) - - -class AgentConfigDict(TypedDict, total=False): - """Configuration for an Agent.""" - - tools_text: Optional[str] - """A JSON string containing a list of tools available to an agent.""" - - tools: Optional[ToolsDict] - """List of tools.""" - - developer_instruction: Optional[InstanceDataDict] - """A field containing instructions from the developer for the agent.""" - - -AgentConfigOrDict = Union[AgentConfig, AgentConfigDict] - - -class Events(_common.BaseModel): - """Represents a list of events for an agent.""" - - event: Optional[list[genai_types.Content]] = Field( - default=None, description="""A list of events.""" - ) - - -class EventsDict(TypedDict, total=False): - """Represents a list of events for an agent.""" - - event: Optional[list[genai_types.ContentDict]] - """A list of events.""" - - -EventsOrDict = Union[Events, EventsDict] - - -class AgentData(_common.BaseModel): - """Contains data specific to agent evaluations.""" - - agent_config: Optional[AgentConfig] = Field( - default=None, description="""Agent configuration.""" - ) - events_text: Optional[str] = Field( - default=None, description="""A JSON string containing a sequence of events.""" - ) - events: Optional[Events] = Field(default=None, description="""A list of events.""") - - -class AgentDataDict(TypedDict, total=False): - """Contains data specific to agent evaluations.""" - - agent_config: Optional[AgentConfigDict] - """Agent configuration.""" - - events_text: Optional[str] - """A JSON string containing a sequence of events.""" - - events: Optional[EventsDict] - """A list of events.""" - - -AgentDataOrDict = Union[AgentData, AgentDataDict] - - class EvaluationInstance(_common.BaseModel): """A single instance to be evaluated.""" - prompt: Optional[InstanceData] = Field( + prompt: Optional[evals_types.InstanceData] = Field( default=None, description="""Data used to populate placeholder `prompt` in a metric prompt template.""", ) - response: Optional[InstanceData] = Field( + response: Optional[evals_types.InstanceData] = Field( default=None, description="""Data used to populate placeholder `response` in a metric prompt template.""", ) - reference: Optional[InstanceData] = Field( + reference: Optional[evals_types.InstanceData] = Field( default=None, description="""Data used to populate placeholder `reference` in a metric prompt template.""", ) @@ -2840,7 +2659,7 @@ class EvaluationInstance(_common.BaseModel): default=None, description="""Other data used to populate placeholders based on their key.""", ) - agent_data: Optional[AgentData] = Field( + agent_data: Optional[evals_types.AgentData] = Field( default=None, description="""Data used for agent evaluation.""" ) rubric_groups: Optional[dict[str, "RubricGroup"]] = Field( @@ -2852,19 +2671,19 @@ class EvaluationInstance(_common.BaseModel): class EvaluationInstanceDict(TypedDict, total=False): """A single instance to be evaluated.""" - prompt: Optional[InstanceDataDict] + prompt: Optional[evals_types.InstanceData] """Data used to populate placeholder `prompt` in a metric prompt template.""" - response: Optional[InstanceDataDict] + response: Optional[evals_types.InstanceData] """Data used to populate placeholder `response` in a metric prompt template.""" - reference: Optional[InstanceDataDict] + reference: Optional[evals_types.InstanceData] """Data used to populate placeholder `reference` in a metric prompt template.""" other_data: Optional[MapInstanceDict] """Other data used to populate placeholders based on their key.""" - agent_data: Optional[AgentDataDict] + agent_data: Optional[evals_types.AgentData] """Data used for agent evaluation.""" rubric_groups: Optional[dict[str, "RubricGroupDict"]] diff --git a/vertexai/_genai/types/evals.py b/vertexai/_genai/types/evals.py new file mode 100644 index 0000000000..9ecade7ee3 --- /dev/null +++ b/vertexai/_genai/types/evals.py @@ -0,0 +1,202 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Code generated by the Google Gen AI SDK generator DO NOT EDIT. + +from typing import Optional, Union +from google.genai import _common +from google.genai import types as genai_types +from pydantic import Field +from typing_extensions import TypedDict + + +class AgentInfo(_common.BaseModel): + """The agent info of an agent, used for agent eval.""" + + agent: Optional[str] = Field( + default=None, + description="""The agent engine used to run agent. Agent engine resource name in str type, with format + `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.""", + ) + name: Optional[str] = Field( + default=None, description="""Agent name, used as an identifier.""" + ) + instruction: Optional[str] = Field( + default=None, description="""Agent developer instruction.""" + ) + description: Optional[str] = Field( + default=None, description="""Agent description.""" + ) + tool_declarations: Optional[genai_types.ToolListUnion] = Field( + default=None, description="""List of tools used by the Agent.""" + ) + + +class AgentInfoDict(TypedDict, total=False): + """The agent info of an agent, used for agent eval.""" + + agent: Optional[str] + """The agent engine used to run agent. Agent engine resource name in str type, with format + `projects/{project}/locations/{location}/reasoningEngines/{reasoning_engine_id}`.""" + + name: Optional[str] + """Agent name, used as an identifier.""" + + instruction: Optional[str] + """Agent developer instruction.""" + + description: Optional[str] + """Agent description.""" + + tool_declarations: Optional[genai_types.ToolListUnionDict] + """List of tools used by the Agent.""" + + +AgentInfoOrDict = Union[AgentInfo, AgentInfoDict] + + +class Tools(_common.BaseModel): + """Represents a list of tools for an agent.""" + + tool: Optional[list[genai_types.Tool]] = Field( + default=None, + description="""List of tools: each tool can have multiple function declarations.""", + ) + + +class ToolsDict(TypedDict, total=False): + """Represents a list of tools for an agent.""" + + tool: Optional[list[genai_types.ToolDict]] + """List of tools: each tool can have multiple function declarations.""" + + +ToolsOrDict = Union[Tools, ToolsDict] + + +class InstanceDataContents(_common.BaseModel): + """List of standard Content messages from Gemini API.""" + + contents: Optional[list[genai_types.Content]] = Field( + default=None, description="""Repeated contents.""" + ) + + +class InstanceDataContentsDict(TypedDict, total=False): + """List of standard Content messages from Gemini API.""" + + contents: Optional[list[genai_types.ContentDict]] + """Repeated contents.""" + + +InstanceDataContentsOrDict = Union[InstanceDataContents, InstanceDataContentsDict] + + +class InstanceData(_common.BaseModel): + """Instance data used to populate placeholders in a metric prompt template.""" + + text: Optional[str] = Field(default=None, description="""Text data.""") + contents: Optional[InstanceDataContents] = Field( + default=None, description="""List of Gemini content data.""" + ) + + +class InstanceDataDict(TypedDict, total=False): + """Instance data used to populate placeholders in a metric prompt template.""" + + text: Optional[str] + """Text data.""" + + contents: Optional[InstanceDataContentsDict] + """List of Gemini content data.""" + + +InstanceDataOrDict = Union[InstanceData, InstanceDataDict] + + +class AgentConfig(_common.BaseModel): + """Configuration for an Agent.""" + + tools_text: Optional[str] = Field( + default=None, + description="""A JSON string containing a list of tools available to an agent.""", + ) + tools: Optional[Tools] = Field(default=None, description="""List of tools.""") + developer_instruction: Optional[InstanceData] = Field( + default=None, + description="""A field containing instructions from the developer for the agent.""", + ) + + +class AgentConfigDict(TypedDict, total=False): + """Configuration for an Agent.""" + + tools_text: Optional[str] + """A JSON string containing a list of tools available to an agent.""" + + tools: Optional[ToolsDict] + """List of tools.""" + + developer_instruction: Optional[InstanceDataDict] + """A field containing instructions from the developer for the agent.""" + + +AgentConfigOrDict = Union[AgentConfig, AgentConfigDict] + + +class Events(_common.BaseModel): + """Represents a list of events for an agent.""" + + event: Optional[list[genai_types.Content]] = Field( + default=None, description="""A list of events.""" + ) + + +class EventsDict(TypedDict, total=False): + """Represents a list of events for an agent.""" + + event: Optional[list[genai_types.ContentDict]] + """A list of events.""" + + +EventsOrDict = Union[Events, EventsDict] + + +class AgentData(_common.BaseModel): + """Contains data specific to agent evaluations.""" + + agent_config: Optional[AgentConfig] = Field( + default=None, description="""Agent configuration.""" + ) + events_text: Optional[str] = Field( + default=None, description="""A JSON string containing a sequence of events.""" + ) + events: Optional[Events] = Field(default=None, description="""A list of events.""") + + +class AgentDataDict(TypedDict, total=False): + """Contains data specific to agent evaluations.""" + + agent_config: Optional[AgentConfigDict] + """Agent configuration.""" + + events_text: Optional[str] + """A JSON string containing a sequence of events.""" + + events: Optional[EventsDict] + """A list of events.""" + + +AgentDataOrDict = Union[AgentData, AgentDataDict]