Upgrade delta-sharing-client to 1.0.5 (#2955)

<!-- Thanks for sending a pull request! Here are some tips for you: 1. If this is your first time, please read our contributor guidelines: https://github.com/delta-io/delta/blob/master/CONTRIBUTING.md 2. If the PR is unfinished, add '[WIP]' in your PR title, e.g., '[WIP] Your PR title ...'. 3. Be sure to keep the PR description updated to reflect all changes. 4. Please write your PR title to summarize what this PR proposes. 5. If possible, provide a concise example to reproduce the issue for a faster review. 6. If applicable, include the corresponding issue number in the PR title and link it in the body. --> #### Which Delta project/connector is this regarding? <!-- Please add the component selected below to the beginning of the pull request title For example: [Spark] Title of my pull request --> - [ ] Spark - [ ] Standalone - [ ] Flink - [ ] Kernel - [X] Other (Delta Sharing) ## Description Upgrade delta-sharing-client to 1.0.5 ## How was this patch tested? Unit Tests ## Does this PR introduce _any_ user-facing changes? No

Author: linzhou-db

build.sbt

@@ -310,7 +310,7 @@ lazy val sharing = (project in file("sharing"))
  libraryDependencies ++= Seq(
  "org.apache.spark" %% "spark-sql" % defaultSparkVersion % "provided",
 
- "io.delta" %% "delta-sharing-client" % "1.0.4",
+ "io.delta" %% "delta-sharing-client" % "1.0.5",
 
  // Test deps
  "org.scalatest" %% "scalatest" % scalaTestVersion % "test",

sharing/src/main/scala/io/delta/sharing/spark/DeltaFormatSharingSource.scala

@@ -34,6 +34,7 @@ import org.apache.spark.sql.delta.sources.{
  DeltaSourceOffset
 }
 import io.delta.sharing.client.DeltaSharingClient
+import io.delta.sharing.client.util.ConfUtils
 import io.delta.sharing.client.model.{Table => DeltaSharingTable}
 
 import org.apache.spark.delta.sharing.CachedTableManager
@@ -155,7 +156,17 @@ case class DeltaFormatSharingSource(
  private var lastTimestampForGetVersionFromServer: Long = -1
 
  // The minimum gap between two getTableVersion rpcs, to avoid a high traffic load to the server.
- private val QUERY_TABLE_VERSION_INTERVAL_MILLIS = TimeUnit.SECONDS.toMillis(30)
+ private val QUERY_TABLE_VERSION_INTERVAL_MILLIS = {
+ val intervalSeconds = ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS.max(
+ ConfUtils.streamingQueryTableVersionIntervalSeconds(spark.sessionState.conf)
+ )
+ logInfo(s"Configured queryTableVersionIntervalSeconds:${intervalSeconds}.")
+ if (intervalSeconds < ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS) {
+ throw new IllegalArgumentException(s"QUERY_TABLE_VERSION_INTERVAL_MILLIS($intervalSeconds) " +
+ s"must not be less than ${ConfUtils.MINIMUM_TABLE_VERSION_INTERVAL_SECONDS} seconds.")
+ }
+ TimeUnit.SECONDS.toMillis(intervalSeconds)
+ }
 
  // Maximum number of versions of getFiles() rpc when fetching files from the server. Used to
  // reduce the number of files returned to avoid timeout of the rpc on the server.

sharing/src/test/scala/io/delta/sharing/spark/DeltaFormatSharingSourceSuite.scala

@@ -215,6 +215,23 @@ class DeltaFormatSharingSourceSuite
  .load(tablePath)
  .filter($"value" contains "keep")
 
+ spark.sessionState.conf.setConfString(
+ "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds",
+ "9s"
+ )
+ val e = intercept[Exception] {
+ testStream(df)(
+ AssertOnQuery { q =>
+ q.processAllAvailable(); true
+ }
+ )
+ }
+ assert(e.getMessage.contains("must not be less than 10 seconds"))
+
+ spark.sessionState.conf.setConfString(
+ "spark.delta.sharing.streaming.queryTableVersionIntervalSeconds",
+ "10s"
+ )
  testStream(df)(
  AssertOnQuery { q =>
  q.processAllAvailable(); true