Copybara import of the project:

-- 73bda4c by Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>: feat: Introduce RagFileMetadataConfig for importing metadata to Rag PiperOrigin-RevId: 770274285 Source-Link: googleapis/googleapis@4cdc2aa Source-Link: googleapis/googleapis-gen@9dbe3e0 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiOWRiZTNlMGYyZGM5NTljYzI2YzU5NGM3NGE5YTkzZTQwOGMxNThiNiJ9 -- 08e37c3 by Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>: feat: add EncryptionSpec field for RagCorpus CMEK feature to v1 PiperOrigin-RevId: 770837205 Source-Link: googleapis/googleapis@3a45aa3 Source-Link: googleapis/googleapis-gen@6d6b54f Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiNmQ2YjU0ZmMzZTExYmQ3OWM1MjBhNGRmOGNiMTU2MWEyYzZhYTE0OSJ9 -- 2e54a68 by Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>: 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md COPYBARA_INTEGRATE_REVIEW=#5422 from googleapis:owl-bot-copy 8660dc9 PiperOrigin-RevId: 771240575

Author: gcf-owl-bot[bot]

google/cloud/aiplatform_v1/services/migration_service/client.py

@@ -265,40 +265,40 @@ def parse_dataset_path(path: str) -> Dict[str, str]:
  @staticmethod
  def dataset_path(
  project: str,
- location: str,
  dataset: str,
  ) -> str:
  """Returns a fully-qualified dataset string."""
- return "projects/{project}/locations/{location}/datasets/{dataset}".format(
+ return "projects/{project}/datasets/{dataset}".format(
  project=project,
- location=location,
  dataset=dataset,
  )
 
  @staticmethod
  def parse_dataset_path(path: str) -> Dict[str, str]:
  """Parses a dataset path into its component segments."""
- m = re.match(
- r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/datasets/(?P<dataset>.+?)$",
- path,
- )
+ m = re.match(r"^projects/(?P<project>.+?)/datasets/(?P<dataset>.+?)$", path)
  return m.groupdict() if m else {}
 
  @staticmethod
  def dataset_path(
  project: str,
+ location: str,
  dataset: str,
  ) -> str:
  """Returns a fully-qualified dataset string."""
- return "projects/{project}/datasets/{dataset}".format(
+ return "projects/{project}/locations/{location}/datasets/{dataset}".format(
  project=project,
+ location=location,
  dataset=dataset,
  )
 
  @staticmethod
  def parse_dataset_path(path: str) -> Dict[str, str]:
  """Parses a dataset path into its component segments."""
- m = re.match(r"^projects/(?P<project>.+?)/datasets/(?P<dataset>.+?)$", path)
+ m = re.match(
+ r"^projects/(?P<project>.+?)/locations/(?P<location>.+?)/datasets/(?P<dataset>.+?)$",
+ path,
+ )
  return m.groupdict() if m else {}
 
  @staticmethod

google/cloud/aiplatform_v1/services/vertex_rag_data_service/async_client.py

@@ -48,6 +48,7 @@
 from google.api_core import operation as gac_operation # type: ignore
 from google.api_core import operation_async # type: ignore
 from google.cloud.aiplatform_v1.services.vertex_rag_data_service import pagers
+from google.cloud.aiplatform_v1.types import encryption_spec
 from google.cloud.aiplatform_v1.types import io
 from google.cloud.aiplatform_v1.types import operation as gca_operation
 from google.cloud.aiplatform_v1.types import vertex_rag_data

google/cloud/aiplatform_v1/services/vertex_rag_data_service/client.py

@@ -64,6 +64,7 @@
 from google.api_core import operation as gac_operation # type: ignore
 from google.api_core import operation_async # type: ignore
 from google.cloud.aiplatform_v1.services.vertex_rag_data_service import pagers
+from google.cloud.aiplatform_v1.types import encryption_spec
 from google.cloud.aiplatform_v1.types import io
 from google.cloud.aiplatform_v1.types import operation as gca_operation
 from google.cloud.aiplatform_v1.types import vertex_rag_data

google/cloud/aiplatform_v1/types/tuning_job.py

@@ -529,9 +529,13 @@ class SupervisedTuningSpec(proto.Message):
 
  Attributes:
  training_dataset_uri (str):
- Required. Training dataset used for tuning. The dataset can be specified as either a Cloud Storage path to a JSONL file or as the resource name of a Vertex Multimodal Dataset.
+ Required. Cloud Storage path to file
+ containing training dataset for tuning. The
+ dataset must be formatted as a JSONL file.
  validation_dataset_uri (str):
- Optional. Validation dataset used for tuning. The dataset can be specified as either a Cloud Storage path to a JSONL file or as the resource name of a Vertex Multimodal Dataset.
+ Optional. Cloud Storage path to file
+ containing validation dataset for tuning. The
+ dataset must be formatted as a JSONL file.
  hyper_parameters (google.cloud.aiplatform_v1.types.SupervisedHyperParameters):
  Optional. Hyperparameters for SFT.
  export_last_checkpoint_only (bool):

google/cloud/aiplatform_v1/types/vertex_rag_data.py

@@ -20,6 +20,7 @@
 import proto # type: ignore
 
 from google.cloud.aiplatform_v1.types import api_auth as gca_api_auth
+from google.cloud.aiplatform_v1.types import encryption_spec as gca_encryption_spec
 from google.cloud.aiplatform_v1.types import io
 from google.protobuf import timestamp_pb2 # type: ignore
 
@@ -414,6 +415,12 @@ class RagCorpus(proto.Message):
  was last updated.
  corpus_status (google.cloud.aiplatform_v1.types.CorpusStatus):
  Output only. RagCorpus state.
+ encryption_spec (google.cloud.aiplatform_v1.types.EncryptionSpec):
+ Optional. Immutable. The CMEK key name used
+ to encrypt at-rest data related to this Corpus.
+ Only applicable to RagManagedDb option for
+ Vector DB. This field can only be set at corpus
+ creation time, and cannot be updated or deleted.
  """
 
  vector_db_config: "RagVectorDbConfig" = proto.Field(
@@ -455,6 +462,11 @@ class RagCorpus(proto.Message):
  number=8,
  message="CorpusStatus",
  )
+ encryption_spec: gca_encryption_spec.EncryptionSpec = proto.Field(
+ proto.MESSAGE,
+ number=12,
+ message=gca_encryption_spec.EncryptionSpec,
+ )
 
 
 class RagFile(proto.Message):

google/cloud/aiplatform_v1beta1/__init__.py

@@ -1202,6 +1202,7 @@
 from .types.vertex_rag_data import RagEngineConfig
 from .types.vertex_rag_data import RagFile
 from .types.vertex_rag_data import RagFileChunkingConfig
+from .types.vertex_rag_data import RagFileMetadataConfig
 from .types.vertex_rag_data import RagFileParsingConfig
 from .types.vertex_rag_data import RagFileTransformationConfig
 from .types.vertex_rag_data import RagManagedDbConfig
@@ -2128,6 +2129,7 @@
  "RagEngineConfig",
  "RagFile",
  "RagFileChunkingConfig",
+ "RagFileMetadataConfig",
  "RagFileParsingConfig",
  "RagFileTransformationConfig",
  "RagManagedDbConfig",

google/cloud/aiplatform_v1beta1/types/__init__.py

@@ -1355,6 +1355,7 @@
  RagEngineConfig,
  RagFile,
  RagFileChunkingConfig,
+ RagFileMetadataConfig,
  RagFileParsingConfig,
  RagFileTransformationConfig,
  RagManagedDbConfig,
@@ -2499,6 +2500,7 @@
  "RagEngineConfig",
  "RagFile",
  "RagFileChunkingConfig",
+ "RagFileMetadataConfig",
  "RagFileParsingConfig",
  "RagFileTransformationConfig",
  "RagManagedDbConfig",

google/cloud/aiplatform_v1beta1/types/tuning_job.py

@@ -764,9 +764,14 @@ class SupervisedTuningSpec(proto.Message):
 
  Attributes:
  training_dataset_uri (str):
- Required. Training dataset used for tuning. The dataset can be specified as either a Cloud Storage path to a JSONL file or as the resource name of a Vertex Multimodal Dataset.
+ Required. Cloud Storage path to file
+ containing training dataset for tuning. The
+ dataset must be formatted as a JSONL file.
  validation_dataset_uri (str):
- Optional. Validation dataset used for tuning. The dataset can be specified as either a Cloud Storage path to a JSONL file or as the resource name of a Vertex Multimodal Dataset.a1.types.SupervisedHyperParameters):
+ Optional. Cloud Storage path to file
+ containing validation dataset for tuning. The
+ dataset must be formatted as a JSONL file.
+ hyper_parameters (google.cloud.aiplatform_v1beta1.types.SupervisedHyperParameters):
  Optional. Hyperparameters for SFT.
  export_last_checkpoint_only (bool):
  Optional. If set to true, disable

google/cloud/aiplatform_v1beta1/types/vertex_rag_data.py

@@ -39,6 +39,7 @@
  "RagFileChunkingConfig",
  "RagFileTransformationConfig",
  "RagFileParsingConfig",
+ "RagFileMetadataConfig",
  "UploadRagFileConfig",
  "ImportRagFilesConfig",
  "RagManagedDbConfig",
@@ -776,6 +777,9 @@ class RagFile(proto.Message):
  last updated.
  file_status (google.cloud.aiplatform_v1beta1.types.FileStatus):
  Output only. State of the RagFile.
+ user_metadata (str):
+ Output only. The metadata for metadata
+ search. The contents will be be in JSON format.
  """
 
  class RagFileType(proto.Enum):
@@ -865,6 +869,10 @@ class RagFileType(proto.Enum):
  number=13,
  message="FileStatus",
  )
+ user_metadata: str = proto.Field(
+ proto.STRING,
+ number=15,
+ )
 
 
 class RagChunk(proto.Message):
@@ -1136,6 +1144,103 @@ class LlmParser(proto.Message):
  )
 
 
+class RagFileMetadataConfig(proto.Message):
+ r"""Metadata config for RagFile.
+
+ This message has `oneof`_ fields (mutually exclusive fields).
+ For each oneof, at most one member field can be set at the same time.
+ Setting any member of the oneof automatically clears all other
+ members.
+
+ .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields
+
+ Attributes:
+ gcs_metadata_schema_source (google.cloud.aiplatform_v1beta1.types.GcsSource):
+ Google Cloud Storage location. Supports importing individual
+ files as well as entire Google Cloud Storage directories.
+ Sample formats:
+
+ - ``gs://bucket_name/my_directory/object_name/metadata_schema.json``
+ - ``gs://bucket_name/my_directory`` If providing a
+ directory, the metadata schema will be read from the
+ files that ends with "metadata_schema.json" in the
+ directory.
+
+ This field is a member of `oneof`_ ``metadata_schema_source``.
+ google_drive_metadata_schema_source (google.cloud.aiplatform_v1beta1.types.GoogleDriveSource):
+ Google Drive location. Supports importing individual files
+ as well as Google Drive folders. If providing a folder, the
+ metadata schema will be read from the files that ends with
+ "metadata_schema.json" in the directory.
+
+ This field is a member of `oneof`_ ``metadata_schema_source``.
+ inline_metadata_schema_source (str):
+ Inline metadata schema source. Must be a JSON
+ string.
+
+ This field is a member of `oneof`_ ``metadata_schema_source``.
+ gcs_metadata_source (google.cloud.aiplatform_v1beta1.types.GcsSource):
+ Google Cloud Storage location. Supports importing individual
+ files as well as entire Google Cloud Storage directories.
+ Sample formats:
+
+ - ``gs://bucket_name/my_directory/object_name/metadata.json``
+ - ``gs://bucket_name/my_directory`` If providing a
+ directory, the metadata will be read from the files that
+ ends with "metadata.json" in the directory.
+
+ This field is a member of `oneof`_ ``metadata_source``.
+ google_drive_metadata_source (google.cloud.aiplatform_v1beta1.types.GoogleDriveSource):
+ Google Drive location. Supports importing
+ individual files as well as Google Drive
+ folders. If providing a directory, the metadata
+ will be read from the files that ends with
+ "metadata.json" in the directory.
+
+ This field is a member of `oneof`_ ``metadata_source``.
+ inline_metadata_source (str):
+ Inline metadata source. Must be a JSON
+ string.
+
+ This field is a member of `oneof`_ ``metadata_source``.
+ """
+
+ gcs_metadata_schema_source: io.GcsSource = proto.Field(
+ proto.MESSAGE,
+ number=1,
+ oneof="metadata_schema_source",
+ message=io.GcsSource,
+ )
+ google_drive_metadata_schema_source: io.GoogleDriveSource = proto.Field(
+ proto.MESSAGE,
+ number=2,
+ oneof="metadata_schema_source",
+ message=io.GoogleDriveSource,
+ )
+ inline_metadata_schema_source: str = proto.Field(
+ proto.STRING,
+ number=3,
+ oneof="metadata_schema_source",
+ )
+ gcs_metadata_source: io.GcsSource = proto.Field(
+ proto.MESSAGE,
+ number=4,
+ oneof="metadata_source",
+ message=io.GcsSource,
+ )
+ google_drive_metadata_source: io.GoogleDriveSource = proto.Field(
+ proto.MESSAGE,
+ number=5,
+ oneof="metadata_source",
+ message=io.GoogleDriveSource,
+ )
+ inline_metadata_source: str = proto.Field(
+ proto.STRING,
+ number=6,
+ oneof="metadata_source",
+ )
+
+
 class UploadRagFileConfig(proto.Message):
  r"""Config for uploading RagFile.
 
@@ -1146,6 +1251,15 @@ class UploadRagFileConfig(proto.Message):
  rag_file_transformation_config (google.cloud.aiplatform_v1beta1.types.RagFileTransformationConfig):
  Specifies the transformation config for
  RagFiles.
+ rag_file_metadata_config (google.cloud.aiplatform_v1beta1.types.RagFileMetadataConfig):
+ Specifies the metadata config for RagFiles.
+ Including paths for metadata schema and
+ metadata. Alteratively, inline metadata schema
+ and metadata can be provided.
+ rag_file_parsing_config (google.cloud.aiplatform_v1beta1.types.RagFileParsingConfig):
+ Optional. Specifies the parsing config for
+ RagFiles. RAG will use the default parser if
+ this field is not set.
  """
 
  rag_file_chunking_config: "RagFileChunkingConfig" = proto.Field(
@@ -1158,6 +1272,16 @@ class UploadRagFileConfig(proto.Message):
  number=3,
  message="RagFileTransformationConfig",
  )
+ rag_file_metadata_config: "RagFileMetadataConfig" = proto.Field(
+ proto.MESSAGE,
+ number=4,
+ message="RagFileMetadataConfig",
+ )
+ rag_file_parsing_config: "RagFileParsingConfig" = proto.Field(
+ proto.MESSAGE,
+ number=5,
+ message="RagFileParsingConfig",
+ )
 
 
 class ImportRagFilesConfig(proto.Message):
@@ -1241,6 +1365,10 @@ class ImportRagFilesConfig(proto.Message):
  Optional. Specifies the parsing config for
  RagFiles. RAG will use the default parser if
  this field is not set.
+ rag_file_metadata_config (google.cloud.aiplatform_v1beta1.types.RagFileMetadataConfig):
+ Specifies the metadata config for RagFiles.
+ Including paths for metadata schema and
+ metadata.
  max_embedding_requests_per_min (int):
  Optional. The max number of queries per
  minute that this job is allowed to make to the
@@ -1338,6 +1466,11 @@ class ImportRagFilesConfig(proto.Message):
  number=8,
  message="RagFileParsingConfig",
  )
+ rag_file_metadata_config: "RagFileMetadataConfig" = proto.Field(
+ proto.MESSAGE,
+ number=17,
+ message="RagFileMetadataConfig",
+ )
  max_embedding_requests_per_min: int = proto.Field(
  proto.INT32,
  number=5,

samples/generated_samples/snippet_metadata_google.cloud.aiplatform.v1.json

@@ -8,7 +8,7 @@
  ],
  "language": "PYTHON",
  "name": "google-cloud-aiplatform",
- "version": "1.97.0"
+ "version": "0.1.0"
  },
  "snippets": [
  {