feat: Modernized Bigtable Admin Client featuring selective GAPIC generation (#1177)

* chore: Removed old admin_v2 GAPIC layer (#1111) * feat!: Generated Selective GAPIC layer for Admin API (#1112) * chore: Updated service YAML by making all methods in BigtableInstanceAdmin public (#1113) * refactor: Refactored classic client to use new Admin API (#1114) * refactor: Refactored classic client to use new Admin API * added newline after gapic_version files * fix: Made generate_consistency_token and check_consistency public (#1116) methods * feat: Consistency polling + restore table for sync client in admin (#1117) * feat: Prototyped handwritten layer * Added newlines * linting * Added docstrings for restore table and consistency token polling; removed gc_rule * docs: owlbot related changes (#1133) * docs: owlbot related changes * Addressed PR feedback + made changes to toc.yml for docs pipeline * Fixed type hint * linting + added validation for admin section * linting + added noqas to owlbot lines * tests: Tests for sync client + fixes + client library versioning (#1132) * tests: Tests for sync client + fixes + client library versioning * Removed raise exception * linting + name changes in tests + added test for timeout * linting * Fixed tests on Python 3.7 * feat: Proto-plus modifications for enforcing strict oneofs (#1126) * feat: Proto-plus modifications for enforcing strict oneofs * Added template directory + changed unit tests to pytest * Finished README * linting * Added source of truth comment * feat: Reworked the wait_for_consistency call (#1144) * feat: Reworked the wait_for_consistency call * linting * Update google/cloud/bigtable/admin_v2/overlay/services/bigtable_table_admin/client.py Co-authored-by: Mattie Fu <mattiefu@google.com> * Improved documentation * linting again * linting --------- Co-authored-by: Mattie Fu <mattiefu@google.com> * feat: Async consistency polling harness (#1142) * feat: Async consistency polling harness * Fixed AsyncMock issue in Python 3.7 * Reworked async_consistency and added async client to __init__.py * linting * addressed review feedback * linting * feat: Restore Table LRO rework + async restore table (#1148) * chore(tests): system tests for autogen API (#1151) * tests: system tests for autogen API * Fixed async system tests * addressed review feedback * Fixed system test failure at the end of a test run * Linting * more linting * chore: Moved Admin API from google.cloud.bigtable.admin_v2 back to google.cloud.bigtable_admin_v2 (#1153) * chore: Removed autogenerated files from the feature branch (#1170) * chore: Merged selective GAPIC autogenerated changes into feature branch (#1175) * chore: Merged selective GAPIC owlbot changes into feature branch * linting * changed comment text * Removed redundant items * Fixed owlbot infinitely appending text * Added comments + fixed indentation in Owlbot * Added anonymous credentials to client tests * Fixed project ID issue in system tests * Fixed docstrings and skipped system tests on emulator. --------- Co-authored-by: Mattie Fu <mattiefu@google.com>

Author: gkevinzheng

docs/admin_client/admin_client_usage.rst

@@ -0,0 +1,11 @@
+Admin Client
+============
+.. toctree::
+ :maxdepth: 2
+
+ services_
+ types_
+
+..
+ This should be the only handwritten RST file in this directory.
+ Everything else should be autogenerated.

docs/admin_client/bigtable_instance_admin.rst

@@ -0,0 +1,10 @@
+BigtableInstanceAdmin
+---------------------------------------
+
+.. automodule:: google.cloud.bigtable_admin_v2.services.bigtable_instance_admin
+ :members:
+ :inherited-members:
+
+.. automodule:: google.cloud.bigtable_admin_v2.services.bigtable_instance_admin.pagers
+ :members:
+ :inherited-members:

docs/admin_client/bigtable_table_admin.rst

@@ -0,0 +1,10 @@
+BigtableTableAdmin
+------------------------------------
+
+.. automodule:: google.cloud.bigtable_admin_v2.overlay.services.bigtable_table_admin
+ :members:
+ :inherited-members:
+
+.. automodule:: google.cloud.bigtable_admin_v2.services.bigtable_table_admin.pagers
+ :members:
+ :inherited-members:

docs/admin_client/services_.rst

@@ -0,0 +1,7 @@
+Services for Google Cloud Bigtable Admin v2 API
+===============================================
+.. toctree::
+ :maxdepth: 2
+
+ bigtable_instance_admin
+ bigtable_table_admin

docs/admin_client/types_.rst

@@ -0,0 +1,10 @@
+Types for Google Cloud Bigtable Admin v2 API
+============================================
+
+.. automodule:: google.cloud.bigtable_admin_v2.types
+ :members:
+ :show-inheritance:
+
+.. automodule:: google.cloud.bigtable_admin_v2.overlay.types
+ :members:
+ :show-inheritance:

docs/index.rst

@@ -9,7 +9,7 @@ Client Types
 
  data_client/data_client_usage
  classic_client/usage
-
+ admin_client/admin_client_usage
 
 Changelog
 ---------

docs/scripts/patch_devsite_toc.py

@@ -20,6 +20,7 @@
 """
 
 
+import glob
 import yaml
 import os
 import shutil
@@ -153,6 +154,81 @@ def copy_markdown(self):
  f"_build/html/docfx_yaml",
  )
 
+ def validate_section(self, toc):
+ # Make sure each rst file is listed in the toc.
+ items_in_toc = [
+ d["items"] for d in toc[0]["items"] if d["name"] == self.title and ".rst"
+ ][0]
+ items_in_dir = [f for f in os.listdir(self.dir_name) if f.endswith(".rst")]
+ # subtract 1 for index
+ assert len(items_in_toc) == len(items_in_dir) - 1
+ for file in items_in_dir:
+ if file != self.index_file_name:
+ base_name, _ = os.path.splitext(file)
+ assert any(d["href"] == f"{base_name}.md" for d in items_in_toc)
+ # make sure the markdown files are present in the docfx_yaml directory
+ md_files = [d["href"] for d in items_in_toc]
+ for file in md_files:
+ assert os.path.exists(f"_build/html/docfx_yaml/{file}")
+
+
+class UIDFilteredTocSection(TocSection):
+ def __init__(self, toc_file_path, section_name, title, uid_prefix):
+ """Creates a filtered section denoted by section_name in the toc_file_path to items with the given UID prefix.
+
+ The section is then renamed to the title.
+ """
+ current_toc = yaml.safe_load(open(toc_file_path, "r"))
+ self.uid_prefix = uid_prefix
+
+ # Since we are looking for a specific section_name there should only
+ # be one match.
+ section_items = [
+ d for d in current_toc[0]["items"] if d["name"] == section_name
+ ][0]["items"]
+ filtered_items = [d for d in section_items if d["uid"].startswith(uid_prefix)]
+ self.items = filtered_items
+ self.title = title
+
+ def copy_markdown(self):
+ """
+ No-op because we are filtering on UIDs, not markdown files.
+ """
+ pass
+
+ def validate_section(self, toc):
+ uids_in_toc = set()
+
+ # A UID-filtered TOC tree looks like the following:
+ # - items:
+ # <optional> items: <more stuff>
+ # name: <name>
+ # uid: <fully qualified path to a class>
+ #
+ # Walk through the TOC tree to find all UIDs recursively.
+ def find_uids_in_items(items):
+ uids_in_toc.add(items["uid"])
+ for subitem in items.get("items", []):
+ find_uids_in_items(subitem)
+
+ items_in_toc = [d["items"] for d in toc[0]["items"] if d["name"] == self.title][
+ 0
+ ]
+ for item in items_in_toc:
+ find_uids_in_items(item)
+
+ # Now that we have all the UIDs, first match all of them
+ # with corresponding .yml files.
+ for uid in uids_in_toc:
+ assert os.path.exists(f"_build/html/docfx_yaml/{uid}.yml")
+
+ # Also validate that every uid yml file that starts with the uid_prefix
+ # exists in the section.
+ for filename in glob.glob(
+ f"{self.uid_prefix}*.yml", root_dir="_build/html/docfx_yaml"
+ ):
+ assert filename[:-4] in uids_in_toc
+
 
 def validate_toc(toc_file_path, expected_section_list, added_sections):
  current_toc = yaml.safe_load(open(toc_file_path, "r"))
@@ -164,43 +240,27 @@ def validate_toc(toc_file_path, expected_section_list, added_sections):
  # make sure each customs ection is in the toc
  for section in added_sections:
  assert section.title in found_sections
- # make sure each rst file in each custom section dir is listed in the toc
- for section in added_sections:
- items_in_toc = [
- d["items"]
- for d in current_toc[0]["items"]
- if d["name"] == section.title and ".rst"
- ][0]
- items_in_dir = [f for f in os.listdir(section.dir_name) if f.endswith(".rst")]
- # subtract 1 for index
- assert len(items_in_toc) == len(items_in_dir) - 1
- for file in items_in_dir:
- if file != section.index_file_name:
- base_name, _ = os.path.splitext(file)
- assert any(d["href"] == f"{base_name}.md" for d in items_in_toc)
- # make sure the markdown files are present in the docfx_yaml directory
- for section in added_sections:
- items_in_toc = [
- d["items"]
- for d in current_toc[0]["items"]
- if d["name"] == section.title and ".rst"
- ][0]
- md_files = [d["href"] for d in items_in_toc]
- for file in md_files:
- assert os.path.exists(f"_build/html/docfx_yaml/{file}")
+ section.validate_section(current_toc)
  print("Toc validation passed")
 
 
 if __name__ == "__main__":
  # Add secrtions for the async_data_client and classic_client directories
  toc_path = "_build/html/docfx_yaml/toc.yml"
+
  custom_sections = [
  TocSection(dir_name="data_client", index_file_name="data_client_usage.rst"),
+ UIDFilteredTocSection(
+ toc_file_path=toc_path,
+ section_name="Bigtable Admin V2",
+ title="Admin Client",
+ uid_prefix="google.cloud.bigtable_admin_v2",
+ ),
  TocSection(dir_name="classic_client", index_file_name="usage.rst"),
  ]
  add_sections(toc_path, custom_sections)
  # Remove the Bigtable section, since it has duplicated data
- remove_sections(toc_path, ["Bigtable"])
+ remove_sections(toc_path, ["Bigtable", "Bigtable Admin V2"])
  # run validation to make sure yaml is structured as we expect
  validate_toc(
  toc_file_path=toc_path,
@@ -210,6 +270,7 @@ def validate_toc(toc_file_path, expected_section_list, added_sections):
  "Changelog",
  "Multiprocessing",
  "Data Client",
+ "Admin Client",
  "Classic Client",
  ],
  added_sections=custom_sections,

google/cloud/bigtable/backup.py

@@ -17,7 +17,7 @@
 import re
 
 from google.cloud._helpers import _datetime_to_pb_timestamp # type: ignore
-from google.cloud.bigtable_admin_v2 import BigtableTableAdminClient
+from google.cloud.bigtable_admin_v2 import BaseBigtableTableAdminClient
 from google.cloud.bigtable_admin_v2.types import table
 from google.cloud.bigtable.encryption_info import EncryptionInfo
 from google.cloud.bigtable.policy import Policy
@@ -106,7 +106,7 @@ def name(self):
  if not self._cluster:
  raise ValueError('"cluster" parameter must be set')
 
- return BigtableTableAdminClient.backup_path(
+ return BaseBigtableTableAdminClient.backup_path(
  project=self._instance._client.project,
  instance=self._instance.instance_id,
  cluster=self._cluster,
@@ -141,7 +141,7 @@ def parent(self):
  :returns: A full path to the parent cluster.
  """
  if not self._parent and self._cluster:
- self._parent = BigtableTableAdminClient.cluster_path(
+ self._parent = BaseBigtableTableAdminClient.cluster_path(
  project=self._instance._client.project,
  instance=self._instance.instance_id,
  cluster=self._cluster,
@@ -163,7 +163,7 @@ def source_table(self):
  :returns: The Table name.
  """
  if not self._source_table and self.table_id:
- self._source_table = BigtableTableAdminClient.table_path(
+ self._source_table = BaseBigtableTableAdminClient.table_path(
  project=self._instance._client.project,
  instance=self._instance.instance_id,
  table=self.table_id,
@@ -226,7 +226,7 @@ def size_bytes(self):
  def state(self):
  """The current state of this Backup.
 
- :rtype: :class:`~google.cloud.bigtable_admin_v2.gapic.enums.Backup.State`
+ :rtype: :class:`~google.cloud.bigtable_admin_v2.types.table.Backup.State`
  :returns: The current state of this Backup.
  """
  return self._state
@@ -305,8 +305,7 @@ def create(self, cluster_id=None):
  created Backup.
 
  :rtype: :class:`~google.api_core.operation.Operation`
- :returns: :class:`~google.cloud.bigtable_admin_v2.types._OperationFuture`
- instance, to be used to poll the status of the 'create' request
+ :returns: A future to be used to poll the status of the 'create' request
  :raises Conflict: if the Backup already exists
  :raises NotFound: if the Instance owning the Backup does not exist
  :raises BadRequest: if the `table` or `expire_time` values are invalid,
@@ -412,7 +411,7 @@ def restore(self, table_id, instance_id=None):
  :param instance_id: (Optional) The ID of the Instance to restore the
  backup into, if different from the current one.
 
- :rtype: :class:`~google.cloud.bigtable_admin_v2.types._OperationFuture`
+ :rtype: :class:`~google.api_core.operation.Operation`
  :returns: A future to be used to poll the status of the 'restore'
  request.
 
@@ -426,14 +425,14 @@ def restore(self, table_id, instance_id=None):
  """
  api = self._instance._client.table_admin_client
  if instance_id:
- parent = BigtableTableAdminClient.instance_path(
+ parent = BaseBigtableTableAdminClient.instance_path(
  project=self._instance._client.project,
  instance=instance_id,
  )
  else:
  parent = self._instance.name
 
- return api.restore_table(
+ return api._restore_table(
  request={"parent": parent, "table_id": table_id, "backup": self.name}
  )
 

google/cloud/bigtable/client.py

@@ -325,11 +325,11 @@ def table_admin_client(self):
  raise ValueError("Client is not an admin client.")
 
  transport = self._create_gapic_client_channel(
- bigtable_admin_v2.BigtableTableAdminClient,
+ bigtable_admin_v2.BaseBigtableTableAdminClient,
  BigtableTableAdminGrpcTransport,
  )
  klass = _create_gapic_client(
- bigtable_admin_v2.BigtableTableAdminClient,
+ bigtable_admin_v2.BaseBigtableTableAdminClient,
  client_options=self._admin_client_options,
  transport=transport,
  )

google/cloud/bigtable/table.py

@@ -47,7 +47,7 @@
 from google.cloud.bigtable.row_set import RowRange
 from google.cloud.bigtable import enums
 from google.cloud.bigtable_v2.types import bigtable as data_messages_v2_pb2
-from google.cloud.bigtable_admin_v2 import BigtableTableAdminClient
+from google.cloud.bigtable_admin_v2 import BaseBigtableTableAdminClient
 from google.cloud.bigtable_admin_v2.types import table as admin_messages_v2_pb2
 from google.cloud.bigtable_admin_v2.types import (
  bigtable_table_admin as table_admin_messages_v2_pb2,
@@ -990,7 +990,7 @@ def list_backups(self, cluster_id=None, filter_=None, order_by=None, page_size=0
  if filter_:
  backups_filter = "({}) AND ({})".format(backups_filter, filter_)
 
- parent = BigtableTableAdminClient.cluster_path(
+ parent = BaseBigtableTableAdminClient.cluster_path(
  project=self._instance._client.project,
  instance=self._instance.instance_id,
  cluster=cluster_id,
@@ -1037,7 +1037,7 @@ def restore(self, new_table_id, cluster_id=None, backup_id=None, backup_name=Non
  and `backup_id` parameters even of such specified.
 
  :return: An instance of
- :class:`~google.cloud.bigtable_admin_v2.types._OperationFuture`.
+ :class:`~google.api_core.operation.Operation`.
 
  :raises: google.api_core.exceptions.AlreadyExists: If the table
  already exists.
@@ -1049,13 +1049,13 @@ def restore(self, new_table_id, cluster_id=None, backup_id=None, backup_name=Non
  """
  api = self._instance._client.table_admin_client
  if not backup_name:
- backup_name = BigtableTableAdminClient.backup_path(
+ backup_name = BaseBigtableTableAdminClient.backup_path(
  project=self._instance._client.project,
  instance=self._instance.instance_id,
  cluster=cluster_id,
  backup=backup_id,
  )
- return api.restore_table(
+ return api._restore_table(
  request={
  "parent": self._instance.name,
  "table_id": new_table_id,