fix: dynamically load BigtableAdmin and BigtableAsyncAdmin (#3341)

* fix: load BigtableAdmin with ProxyFactory * update hbase2_x * fix connection class * reformat * use bytebuddy instead * refactor * fix constructor * update javadoc * update error handling * udpate

Author: mutianf

bigtable-client-core-parent/bigtable-hbase/src/main/java/org/apache/hadoop/hbase/client/AbstractBigtableAdmin.java

@@ -45,6 +45,8 @@
 import com.google.protobuf.ByteString;
 import io.grpc.Status;
 import java.io.IOException;
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
 import java.util.ArrayList;
 import java.util.LinkedList;
 import java.util.List;
@@ -1337,4 +1339,12 @@ public int getMasterInfoPort() throws IOException {
  public void rollWALWriter(ServerName serverName) throws IOException, FailedLogCloseException {
  throw new UnsupportedOperationException("rollWALWriter"); // TODO
  }
+
+ /** Handler for unsupported operations for generating Admin class at runtime. */
+ public static class UnsupportedOperationsHandler implements InvocationHandler {
+ @Override
+ public Object invoke(Object proxy, Method method, Object[] args) throws Throwable {
+ throw new UnsupportedOperationException(method.getName());
+ }
+ }
 }

bigtable-hbase-1.x-parent/bigtable-hbase-1.x-shaded/pom.xml

@@ -159,6 +159,7 @@ limitations under the License.
  <exclude>META-INF/NOTICE</exclude>
  <exclude>META-INF/DEPENDENCIES</exclude>
  <exclude>META-INF/NOTICE.txt</exclude>
+ <exclude>META-INF/versions/9/module-info.class</exclude>
  </excludes>
  </filter>
  </filters>
@@ -331,6 +332,10 @@ limitations under the License.
  <pattern>javax.annotation</pattern>
  <shadedPattern>com.google.bigtable.repackaged.javax.annotation</shadedPattern>
  </relocation>
+ <relocation>
+ <pattern>net.bytebuddy</pattern>
+ <shadedPattern>com.google.bigtable.repackaged.net.bytebuddy</shadedPattern>
+ </relocation>
  </relocations>
  </configuration>
  </execution>

bigtable-hbase-1.x-parent/bigtable-hbase-1.x/pom.xml

@@ -146,6 +146,12 @@ limitations under the License.
  <artifactId>grpc-google-cloud-bigtable-v2</artifactId>
  <scope>compile</scope>
  </dependency>
+ <dependency>
+ <groupId>net.bytebuddy</groupId>
+ <artifactId>byte-buddy</artifactId>
+ <version>1.12.2</version>
+ </dependency>
+
 
  <!-- Test dependencies-->
  <dependency>

bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableAdmin.java

@@ -16,39 +16,31 @@
 package com.google.cloud.bigtable.hbase1_x;
 
 import com.google.api.core.InternalApi;
+import com.google.common.base.Throwables;
 import com.google.common.collect.ImmutableList;
 import com.google.common.util.concurrent.Futures;
 import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.regex.Pattern;
+import net.bytebuddy.ByteBuddy;
+import net.bytebuddy.implementation.InvocationHandlerAdapter;
+import net.bytebuddy.matcher.ElementMatchers;
 import org.apache.hadoop.hbase.ClusterStatus;
-import org.apache.hadoop.hbase.ProcedureInfo;
 import org.apache.hadoop.hbase.ServerName;
 import org.apache.hadoop.hbase.TableExistsException;
 import org.apache.hadoop.hbase.TableName;
 import org.apache.hadoop.hbase.client.AbstractBigtableAdmin;
+import org.apache.hadoop.hbase.client.Admin;
 import org.apache.hadoop.hbase.client.CommonConnection;
-import org.apache.hadoop.hbase.client.LogEntry;
-import org.apache.hadoop.hbase.client.ServerType;
-import org.apache.hadoop.hbase.client.security.SecurityCapability;
-import org.apache.hadoop.hbase.protobuf.generated.AdminProtos;
 import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
 import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
-import org.apache.hadoop.hbase.quotas.QuotaFilter;
-import org.apache.hadoop.hbase.quotas.QuotaRetriever;
-import org.apache.hadoop.hbase.quotas.QuotaSettings;
-import org.apache.hadoop.hbase.snapshot.HBaseSnapshotException;
 import org.apache.hadoop.hbase.snapshot.RestoreSnapshotException;
 import org.apache.hadoop.hbase.snapshot.SnapshotCreationException;
-import org.apache.hadoop.hbase.snapshot.UnknownSnapshotException;
 
 /**
  * This is an hbase 1.x implementation of {@link AbstractBigtableAdmin}. Most methods in this class
@@ -58,7 +50,7 @@
  */
 @InternalApi("For internal usage only")
 @SuppressWarnings("deprecation")
-public class BigtableAdmin extends AbstractBigtableAdmin {
+public abstract class BigtableAdmin extends AbstractBigtableAdmin {
 
  public BigtableAdmin(CommonConnection connection) throws IOException {
  super(connection);
@@ -83,32 +75,12 @@ public void disableTableAsync(TableName tableName) throws IOException {
  disableTable(tableName);
  }
 
- @Override
- public Map<ServerName, Boolean> compactionSwitch(boolean b, List<String> list)
- throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
  /** {@inheritDoc} */
  @Override
  public void enableTableAsync(TableName tableName) throws IOException {
  enableTable(tableName);
  }
 
- /** {@inheritDoc} */
- @Override
- public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionState(TableName tableName)
- throws IOException {
- throw new UnsupportedOperationException("getCompactionState");
- }
-
- /** {@inheritDoc} */
- @Override
- public AdminProtos.GetRegionInfoResponse.CompactionState getCompactionStateForRegion(byte[] bytes)
- throws IOException {
- throw new UnsupportedOperationException("getCompactionStateForRegion");
- }
-
  /**
  * {@inheritDoc}
  *
@@ -123,33 +95,13 @@ public void snapshot(
  snapshot(snapshotId, tableName);
  }
 
- @Override
- public void snapshot(
- String s, TableName tableName, SnapshotDescription.Type type, Map<String, Object> map)
- throws IOException, SnapshotCreationException, IllegalArgumentException {
- throw new UnsupportedOperationException("not implemented");
- }
-
  /** {@inheritDoc} */
  @Override
  public void snapshot(HBaseProtos.SnapshotDescription snapshot)
  throws IOException, SnapshotCreationException, IllegalArgumentException {
  snapshot(snapshot.getName(), TableName.valueOf(snapshot.getTable()));
  }
 
- /** {@inheritDoc} */
- @Override
- public boolean isSnapshotFinished(HBaseProtos.SnapshotDescription snapshot)
- throws IOException, HBaseSnapshotException, UnknownSnapshotException {
- throw new UnsupportedOperationException("isSnapshotFinished"); // TODO
- }
-
- @Override
- public void restoreSnapshot(String s, boolean b, boolean b1)
- throws IOException, RestoreSnapshotException {
- throw new UnsupportedOperationException("restoreSnapshot"); // TODO
- }
-
  @Override
  public void cloneSnapshot(String s, TableName tableName, boolean b)
  throws IOException, TableExistsException, RestoreSnapshotException {
@@ -196,152 +148,6 @@ public List<HBaseProtos.SnapshotDescription> listSnapshots(Pattern pattern) thro
  return response;
  }
 
- @Override
- public List<SnapshotDescription> listTableSnapshots(
- String tableNameRegex, String snapshotNameRegex) throws IOException {
- throw new UnsupportedOperationException("Unsupported - please use listSnapshots");
- }
-
- @Override
- public List<SnapshotDescription> listTableSnapshots(
- Pattern tableNamePattern, Pattern snapshotNamePattern) throws IOException {
- throw new UnsupportedOperationException("Unsupported - please use listSnapshots");
- }
-
- @Override
- public boolean isBalancerEnabled() throws IOException {
- throw new UnsupportedOperationException("isBalancerEnabled"); // TODO
- }
-
- @Override
- public long getLastMajorCompactionTimestamp(TableName tableName) throws IOException {
- throw new UnsupportedOperationException("getLastMajorCompactionTimestamp"); // TODO
- }
-
- @Override
- public long getLastMajorCompactionTimestampForRegion(byte[] regionName) throws IOException {
- throw new UnsupportedOperationException("getLastMajorCompactionTimestampForRegion"); // TODO
- }
-
- @Override
- public void setQuota(QuotaSettings quota) throws IOException {
- throw new UnsupportedOperationException("setQuota"); // TODO
- }
-
- @Override
- public QuotaRetriever getQuotaRetriever(QuotaFilter filter) throws IOException {
- throw new UnsupportedOperationException("getQuotaRetriever"); // TODO
- }
-
- @Override
- public ServerName getMaster() throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public boolean normalize() throws IOException {
- throw new UnsupportedOperationException("normalize"); // TODO
- }
-
- @Override
- public boolean isNormalizerEnabled() throws IOException {
- throw new UnsupportedOperationException("isNormalizerEnabled"); // TODO
- }
-
- @Override
- public boolean setNormalizerRunning(boolean on) throws IOException {
- throw new UnsupportedOperationException("setNormalizerRunning"); // TODO
- }
-
- @Override
- public boolean setCleanerChoreRunning(boolean b) throws IOException {
- throw new UnsupportedOperationException("setCleanerChoreRunning"); // TODO
- }
-
- @Override
- public boolean runCleanerChore() throws IOException {
- throw new UnsupportedOperationException("runCleanerChore"); // TODO
- }
-
- @Override
- public boolean isCleanerChoreEnabled() throws IOException {
- throw new UnsupportedOperationException("isCleanerChoreEnabled"); // TODO
- }
-
- @Override
- public boolean isMasterInMaintenanceMode() throws IOException {
- throw new UnsupportedOperationException("isMasterInMaintenanceMode"); // TODO
- }
-
- @Override
- public boolean abortProcedure(long procId, boolean mayInterruptIfRunning) throws IOException {
- throw new UnsupportedOperationException("abortProcedure"); // TODO
- }
-
- @Override
- public ProcedureInfo[] listProcedures() throws IOException {
- throw new UnsupportedOperationException("listProcedures"); // TODO
- }
-
- @Override
- public Future<Boolean> abortProcedureAsync(long procId, boolean mayInterruptIfRunning)
- throws IOException {
- throw new UnsupportedOperationException("abortProcedureAsync"); // TODO
- }
-
- @Override
- public List<SecurityCapability> getSecurityCapabilities() throws IOException {
- throw new UnsupportedOperationException("getSecurityCapabilities"); // TODO
- }
-
- @Override
- public boolean balancer(boolean arg0) throws IOException {
- throw new UnsupportedOperationException("balancer"); // TODO
- }
-
- @Override
- public boolean isSplitOrMergeEnabled(MasterSwitchType arg0) throws IOException {
- throw new UnsupportedOperationException("isSplitOrMergeEnabled"); // TODO
- }
-
- @Override
- public List<ServerName> listDeadServers() throws IOException {
- throw new UnsupportedOperationException("listDeadServers"); // TODO
- }
-
- @Override
- public List<ServerName> clearDeadServers(List<ServerName> list) throws IOException {
- throw new UnsupportedOperationException("clearDeadServers"); // TODO
- }
-
- @Override
- public boolean snapshotCleanupSwitch(boolean b, boolean b1) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public boolean isSnapshotCleanupEnabled() throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public List<Boolean> clearSlowLogResponses(Set<ServerName> set) throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public List<LogEntry> getLogEntries(
- Set<ServerName> set, String s, ServerType serverType, int i, Map<String, Object> map)
- throws IOException {
- throw new UnsupportedOperationException("not implemented");
- }
-
- @Override
- public boolean[] setSplitOrMergeEnabled(boolean arg0, boolean arg1, MasterSwitchType... arg2)
- throws IOException {
- throw new UnsupportedOperationException("setSplitOrMergeEnabled"); // TODO
- }
-
  @Override
  public ClusterStatus getClusterStatus() throws IOException {
  return new ClusterStatus() {
@@ -352,8 +158,43 @@ public Collection<ServerName> getServers() {
  };
  }
 
- @Override
- public String[] listNamespaces() throws IOException {
- throw new UnsupportedOperationException("not implemented");
+ private static Class<? extends BigtableAdmin> adminClass = null;
+
+ /**
+ * This is a workaround for incompatible changes in hbase minor versions. Dynamically generates a
+ * class that extends BigtableAdmin so incompatible methods won't be accessed unless the methods
+ * are called. If a method is implemented by BigtableAdmin, the generated class will invoke the
+ * implementation in BigtableAdmin. Otherwise it'll throw {@link UnsupportedOperationException}.
+ */
+ private static synchronized Class<? extends BigtableAdmin> getSubclass() {
+ if (adminClass == null) {
+ adminClass =
+ new ByteBuddy()
+ .subclass(BigtableAdmin.class)
+ .method(ElementMatchers.isAbstract())
+ .intercept(
+ InvocationHandlerAdapter.of(
+ new AbstractBigtableAdmin.UnsupportedOperationsHandler()))
+ .make()
+ .load(BigtableAdmin.class.getClassLoader())
+ .getLoaded();
+ }
+ return adminClass;
+ }
+
+ public static Admin createInstance(CommonConnection connection) throws IOException {
+ try {
+ return getSubclass().getDeclaredConstructor(CommonConnection.class).newInstance(connection);
+ } catch (InvocationTargetException e) {
+ // Unwrap and throw IOException or RuntimeException as is, and convert all other exceptions to
+ // IOException because
+ // org.apache.hadoop.hbase.client.Connection#getAdmin() only throws
+ // IOException
+ Throwables.throwIfInstanceOf(e.getTargetException(), IOException.class);
+ Throwables.throwIfInstanceOf(e.getTargetException(), RuntimeException.class);
+ throw new IOException(e);
+ } catch (Exception e) {
+ throw new IOException(e);
+ }
  }
 }

bigtable-hbase-1.x-parent/bigtable-hbase-1.x/src/main/java/com/google/cloud/bigtable/hbase1_x/BigtableConnection.java

@@ -58,7 +58,7 @@ public BigtableConnection(Configuration conf) throws IOException {
  /** {@inheritDoc} */
  @Override
  public Admin getAdmin() throws IOException {
- return new BigtableAdmin(this);
+ return BigtableAdmin.createInstance(this);
  }
 
  @Override