feat: GenAI SDK client(evals) - Add async evaluate_instances method

PiperOrigin-RevId: 784173496

Author: sararob

tests/unit/vertexai/genai/replays/test_batch_evaluate.py

@@ -14,6 +14,8 @@
 #
 # pylint: disable=protected-access,bad-continuation,missing-function-docstring
 
+import pytest
+
 from tests.unit.vertexai.genai.replays import pytest_helper
 from vertexai._genai import types
 
@@ -43,3 +45,24 @@ def test_batch_eval(client):
  globals_for_file=globals(),
  test_method="evals.batch_evaluate",
 )
+
+pytest_plugins = ("pytest_asyncio",)
+
+
+@pytest.mark.asyncio
+async def test_batch_eval_async(client):
+ eval_dataset = types.EvaluationDataset(
+ gcs_source=types.GcsSource(
+ uris=["gs://genai-eval-sdk-replay-test/test_data/inference_results.jsonl"]
+ )
+ )
+
+ response = await client.aio.evals.batch_evaluate(
+ dataset=eval_dataset,
+ metrics=[
+ types.PrebuiltMetric.TEXT_QUALITY,
+ ],
+ dest="gs://genai-eval-sdk-replay-test/test_data/batch_eval_output",
+ )
+ assert "operations" in response.name
+ assert "EvaluateDatasetOperationMetadata" in response.metadata.get("@type")

tests/unit/vertexai/genai/replays/test_evaluate_instances.py

@@ -14,11 +14,12 @@
 #
 # pylint: disable=protected-access,bad-continuation,missing-function-docstring
 
+import json
 
 from tests.unit.vertexai.genai.replays import pytest_helper
 from vertexai._genai import types
 import pandas as pd
-import json
+import pytest
 
 
 def test_bleu_metric(client):
@@ -31,7 +32,11 @@ def test_bleu_metric(client):
  ],
  metric_spec=types.BleuSpec(),
  )
- response = client.evals._evaluate_instances(bleu_input=test_bleu_input)
+ response = client.evals.evaluate_instances(
+ metric_config=types._EvaluateInstancesRequestParameters(
+ bleu_input=test_bleu_input
+ )
+ )
  assert len(response.bleu_results.bleu_metric_values) == 1
 
 
@@ -46,8 +51,10 @@ def test_exact_match_metric(client):
  ],
  metric_spec=types.ExactMatchSpec(),
  )
- response = client.evals._evaluate_instances(
- exact_match_input=test_exact_match_input
+ response = client.evals.evaluate_instances(
+ metric_config=types._EvaluateInstancesRequestParameters(
+ exact_match_input=test_exact_match_input
+ )
  )
  assert len(response.exact_match_results.exact_match_metric_values) == 1
 
@@ -63,7 +70,11 @@ def test_rouge_metric(client):
  ],
  metric_spec=types.RougeSpec(rouge_type="rougeL"),
  )
- response = client.evals._evaluate_instances(rouge_input=test_rouge_input)
+ response = client.evals.evaluate_instances(
+ metric_config=types._EvaluateInstancesRequestParameters(
+ rouge_input=test_rouge_input
+ )
+ )
  assert len(response.rouge_results.rouge_metric_values) == 1
 
 
@@ -78,7 +89,11 @@ def test_pointwise_metric(client):
  metric_prompt_template="Evaluate if the response '{response}' correctly answers the prompt '{prompt}'."
  ),
  )
- response = client.evals._evaluate_instances(pointwise_metric_input=test_input)
+ response = client.evals.evaluate_instances(
+ metric_config=types._EvaluateInstancesRequestParameters(
+ pointwise_metric_input=test_input
+ )
+ )
  assert response.pointwise_metric_result is not None
  assert response.pointwise_metric_result.score is not None
 
@@ -100,8 +115,10 @@ def test_pairwise_metric_with_autorater(client):
  )
  autorater_config = types.AutoraterConfig(sampling_count=2)
 
- response = client.evals._evaluate_instances(
- pairwise_metric_input=test_input, autorater_config=autorater_config
+ response = client.evals.evaluate_instances(
+ metric_config=types._EvaluateInstancesRequestParameters(
+ pairwise_metric_input=test_input, autorater_config=autorater_config
+ )
  )
  assert response.pairwise_metric_result is not None
  assert response.pairwise_metric_result.pairwise_choice is not None
@@ -147,3 +164,25 @@ def test_inference_with_prompt_template(client):
  globals_for_file=globals(),
  test_method="evals.evaluate",
 )
+
+
+pytest_plugins = ("pytest_asyncio",)
+
+
+@pytest.mark.asyncio
+async def test_bleu_metric_async(client):
+ test_bleu_input = types.BleuInput(
+ instances=[
+ types.BleuInstance(
+ reference="The quick brown fox jumps over the lazy dog.",
+ prediction="A fast brown fox leaps over a lazy dog.",
+ )
+ ],
+ metric_spec=types.BleuSpec(),
+ )
+ response = await client.aio.evals.evaluate_instances(
+ metric_config=types._EvaluateInstancesRequestParameters(
+ bleu_input=test_bleu_input
+ )
+ )
+ assert len(response.bleu_results.bleu_metric_values) == 1

vertexai/_genai/evals.py

@@ -1522,3 +1522,21 @@ async def batch_evaluate(
  self._api_client._verify_response(return_value)
 
  return return_value
+
+ async def evaluate_instances(
+ self,
+ *,
+ metric_config: types._EvaluateInstancesRequestParameters,
+ ) -> types.EvaluateInstancesResponse:
+ """Evaluates an instance of a model."""
+
+ if isinstance(metric_config, types._EvaluateInstancesRequestParameters):
+ metric_config = metric_config.model_dump()
+ else:
+ metric_config = dict(metric_config)
+
+ result = await self._evaluate_instances(
+ **metric_config,
+ )
+
+ return result

vertexai/_genai/mypy.ini

@@ -0,0 +1,22 @@
+[mypy]
+# TODO(b/422425982): Fix arg-type errors
+disable_error_code = import-not-found, import-untyped, arg-type
+
+# We only want to run mypy on _genai dir, ignore dependent modules
+[mypy-vertexai.agent_engines.*]
+ignore_errors = True
+
+[mypy-vertexai.preview.*]
+ignore_errors = True
+
+[mypy-vertexai.generative_models.*]
+ignore_errors = True
+
+[mypy-vertexai.prompts.*]
+ignore_errors = True
+
+[mypy-vertexai.tuning.*]
+ignore_errors = True
+
+[mypy-vertexai.caching.*]
+ignore_errors = True