Return read version in ReadTableResult (#198)

Add lombok because pojo refactor was too painful

Author: agilelab-tmnd1991

buildSrc/build.gradle.kts

@@ -10,4 +10,5 @@ dependencies {
  implementation("org.openapi.generator:org.openapi.generator.gradle.plugin:6.6.0")
  implementation("com.diffplug.spotless:spotless-plugin-gradle:6.25.0")
  implementation("com.palantir.gradle.gitversion:gradle-git-version:3.0.0")
+ implementation("io.freefair.lombok:io.freefair.lombok.gradle.plugin:8.4")
 }

buildSrc/src/main/kotlin/whitefox.java-conventions.gradle.kts

@@ -7,6 +7,7 @@ plugins {
  id("com.palantir.git-version")
  id("org.openapi.generator")
  id("com.diffplug.spotless")
+ id("io.freefair.lombok")
 }
 // Projects should use Maven Central for external dependencies
 repositories {

lombok.config

@@ -0,0 +1,2 @@
+lombok.accessors.fluent=true
+lombok.addLombokGeneratedAnnotation = true

protocol/delta-sharing-protocol-api.yml

@@ -413,12 +413,6 @@ paths:
  description: 'Named table'
  schema:
  type: string
- - in: query
- name: startingTimestamp
- required: false
- description: 'Starting Timestamp ISO8601 format, in the UTC timezone'
- schema:
- type: string
  - in: header
  name: delta-sharing-capabilities
  required: false

server/app/src/main/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImpl.java

@@ -164,49 +164,35 @@ share, schema, parseToken(pageToken), Optional.ofNullable(maxResults)),
  exceptionToResponse);
  }
 
- /***
- * Example:
- * TableQueryResponse:
- * type: string
- * example: |
- * {"protocol":{"minReaderVersion":1}}
- * {"metaData":{"id":"f8d5c169-3d01-4ca3-ad9e-7dc3355aedb2","format":{"provider":"parquet"},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"eventTime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"]}}
- * {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-8b0086f2-7b27-4935-ac5a-8ed6215a6640.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=900&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=97b6762cfd8e4d7e94b9d707eff3faf266974f6e7030095c1d4a66350cfd892e","id":"8b0086f2-7b27-4935-ac5a-8ed6215a6640","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"nullCount\":{\"eventTime\":0}}"}}
- * {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-591723a8-6a27-4240-a90e-57426f4736d2.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=899&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=0f7acecba5df7652457164533a58004936586186c56425d9d53c52db574f6b62","id":"591723a8-6a27-4240-a90e-57426f4736d2","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"nullCount\":{\"eventTime\":0}}"}}
- *
- *
- * @param share
- * @param schema
- * @param table
- * @param queryRequest
- * @param startingTimestamp
- * @return
+ /**
+ * <pre>
+ * {"protocol":{"minReaderVersion":1}}
+ * {"metaData":{"id":"f8d5c169-3d01-4ca3-ad9e-7dc3355aedb2","format":{"provider":"parquet"},"schemaString":"{\"type\":\"struct\",\"fields\":[{\"name\":\"eventTime\",\"type\":\"timestamp\",\"nullable\":true,\"metadata\":{}},{\"name\":\"date\",\"type\":\"date\",\"nullable\":true,\"metadata\":{}}]}","partitionColumns":["date"]}}
+ * {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-8b0086f2-7b27-4935-ac5a-8ed6215a6640.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=900&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=97b6762cfd8e4d7e94b9d707eff3faf266974f6e7030095c1d4a66350cfd892e","id":"8b0086f2-7b27-4935-ac5a-8ed6215a6640","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:57.955Z\"},\"nullCount\":{\"eventTime\":0}}"}}
+ * {"file":{"url":"https://<s3-bucket-name>.s3.us-west-2.amazonaws.com/delta-exchange-test/table2/date%3D2021-04-28/part-00000-591723a8-6a27-4240-a90e-57426f4736d2.c000.snappy.parquet?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20210501T010516Z&X-Amz-SignedHeaders=host&X-Amz-Expires=899&X-Amz-Credential=AKIAISZRDL4Q4Q7AIONA%2F20210501%2Fus-west-2%2Fs3%2Faws4_request&X-Amz-Signature=0f7acecba5df7652457164533a58004936586186c56425d9d53c52db574f6b62","id":"591723a8-6a27-4240-a90e-57426f4736d2","partitionValues":{"date":"2021-04-28"},"size":573,"stats":"{\"numRecords\":1,\"minValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"maxValues\":{\"eventTime\":\"2021-04-28T23:33:48.719Z\"},\"nullCount\":{\"eventTime\":0}}"}}
+ * </pre>
  */
  @Override
  public Response queryTable(
  String share,
  String schema,
  String table,
  QueryRequest queryRequest,
- String startingTimestamp,
  String deltaSharingCapabilities) {
  return wrapExceptions(
- () -> optionalToNotFound(
- deltaSharesService.getTableVersion(share, schema, table, startingTimestamp),
- version -> Response.ok(
- tableQueryResponseSerializer.serialize(
- DeltaMappers.readTableResult2api(deltaSharesService.queryTable(
- share,
- schema,
- table,
- DeltaMappers.api2ReadTableRequest(queryRequest)))),
- ndjsonMediaType)
- .header(DELTA_TABLE_VERSION_HEADER, version)
- .header(
- DELTA_SHARE_CAPABILITIES_HEADER,
- getResponseFormatHeader(
- DeltaMappers.toHeaderCapabilitiesMap(deltaSharingCapabilities)))
- .build()),
+ () -> {
+ var readResult = deltaSharesService.queryTable(
+ share, schema, table, DeltaMappers.api2ReadTableRequest(queryRequest));
+ var serializedReadResult =
+ tableQueryResponseSerializer.serialize(DeltaMappers.readTableResult2api(readResult));
+ return Response.ok(serializedReadResult, ndjsonMediaType)
+ .header(DELTA_TABLE_VERSION_HEADER, readResult.version())
+ .header(
+ DELTA_SHARE_CAPABILITIES_HEADER,
+ getResponseFormatHeader(
+ DeltaMappers.toHeaderCapabilitiesMap(deltaSharingCapabilities)))
+ .build();
+ },
  exceptionToResponse);
  }
 

server/app/src/test/java/io/whitefox/api/deltasharing/server/DeltaSharesApiImplTest.java

@@ -352,6 +352,23 @@ public void getTableVersionBadTimestamp() {
  .statusCode(502);
  }
 
+ @DisabledOnOs(OS.WINDOWS)
+ @Test
+ public void queryNotExistingTable() throws IOException {
+ given()
+ .when()
+ .filter(deltaFilter)
+ .body("{}")
+ .header(new Header("Content-Type", "application/json"))
+ .post(
+ "delta-api/v1/shares/{share}/schemas/{schema}/tables/{table}/query",
+ "name",
+ "default",
+ "tableThatDoesNotExist")
+ .then()
+ .statusCode(404);
+ }
+
  @DisabledOnOs(OS.WINDOWS)
  @Test
  public void queryTableCurrentVersion() throws IOException {
@@ -367,6 +384,7 @@ public void queryTableCurrentVersion() throws IOException {
  "table1")
  .then()
  .statusCode(200)
+ .header("Delta-Table-Version", "0")
  .extract()
  .body()
  .asString()
@@ -410,6 +428,7 @@ public void queryTableByVersion() throws IOException {
  "table1")
  .then()
  .statusCode(200)
+ .header("Delta-Table-Version", "0")
  .extract()
  .body()
  .asString()
@@ -453,6 +472,7 @@ public void queryTableByTs() throws IOException {
  "table-with-history")
  .then()
  .statusCode(200)
+ .header("Delta-Table-Version", "0")
  .extract()
  .body()
  .asString()

server/core/src/main/java/io/whitefox/core/ReadTableResult.java

@@ -1,55 +1,12 @@
 package io.whitefox.core;
 
-import io.whitefox.annotations.SkipCoverageGenerated;
 import java.util.List;
-import java.util.Objects;
+import lombok.Value;
 
+@Value
 public class ReadTableResult {
- private final Protocol protocol;
- private final Metadata metadata;
- private final List<TableFile> files;
-
- public ReadTableResult(Protocol protocol, Metadata metadata, List<TableFile> files) {
- this.protocol = protocol;
- this.metadata = metadata;
- this.files = files;
- }
-
- @Override
- @SkipCoverageGenerated
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- ReadTableResult that = (ReadTableResult) o;
- return Objects.equals(protocol, that.protocol)
- && Objects.equals(metadata, that.metadata)
- && Objects.equals(files, that.files);
- }
-
- @Override
- @SkipCoverageGenerated
- public int hashCode() {
- return Objects.hash(protocol, metadata, files);
- }
-
- @Override
- @SkipCoverageGenerated
- public String toString() {
- return "QueryTableResult{" + "protocol="
- + protocol + ", metadata="
- + metadata + ", files="
- + files + '}';
- }
-
- public Protocol protocol() {
- return protocol;
- }
-
- public Metadata metadata() {
- return metadata;
- }
-
- public List<TableFile> files() {
- return files;
- }
+ Protocol protocol;
+ Metadata metadata;
+ List<TableFile> files;
+ long version;
 }

server/core/src/main/java/io/whitefox/core/ReadTableResultToBeSigned.java

@@ -1,56 +1,12 @@
 package io.whitefox.core;
 
-import io.whitefox.annotations.SkipCoverageGenerated;
 import java.util.List;
-import java.util.Objects;
+import lombok.Value;
 
+@Value
 public class ReadTableResultToBeSigned {
- private final Protocol protocol;
- private final Metadata metadata;
- private final List<TableFileToBeSigned> other;
-
- public ReadTableResultToBeSigned(
- Protocol protocol, Metadata metadata, List<TableFileToBeSigned> other) {
- this.protocol = protocol;
- this.metadata = metadata;
- this.other = other;
- }
-
- @Override
- @SkipCoverageGenerated
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
- ReadTableResultToBeSigned that = (ReadTableResultToBeSigned) o;
- return Objects.equals(protocol, that.protocol)
- && Objects.equals(metadata, that.metadata)
- && Objects.equals(other, that.other);
- }
-
- @Override
- @SkipCoverageGenerated
- public int hashCode() {
- return Objects.hash(protocol, metadata, other);
- }
-
- @Override
- @SkipCoverageGenerated
- public String toString() {
- return "QueryTableResult{" + "protocol="
- + protocol + ", metadata="
- + metadata + ", other="
- + other + '}';
- }
-
- public Protocol protocol() {
- return protocol;
- }
-
- public Metadata metadata() {
- return metadata;
- }
-
- public List<TableFileToBeSigned> other() {
- return other;
- }
+ Protocol protocol;
+ Metadata metadata;
+ List<TableFileToBeSigned> other;
+ long version;
 }

server/core/src/main/java/io/whitefox/core/services/DeltaSharedTable.java

@@ -102,7 +102,8 @@ public ReadTableResultToBeSigned queryTable(ReadTableRequest readTableRequest) {
  snapshot.getMetadata().getCreatedTime(),
  f.getStats(),
  f.getPartitionValues()))
- .collect(Collectors.toList()));
+ .collect(Collectors.toList()),
+ snapshot.getVersion());
  }
 
  private Optional<Snapshot> getSnapshot(Optional<String> startingTimestamp) {

server/core/src/main/java/io/whitefox/core/services/DeltaSharesServiceImpl.java

@@ -140,6 +140,7 @@ public ReadTableResult queryTable(
  readTableResultToBeSigned.metadata(),
  readTableResultToBeSigned.other().stream()
  .map(fileSigner::sign)
- .collect(Collectors.toList()));
+ .collect(Collectors.toList()),
+ readTableResultToBeSigned.version());
  }
 }