Update openapi spec and written form to comply with latest delta-sharing protocol (#203)

Author: agilelab-tmnd1991

docsite/docs/protocols/delta-sharing-protocol.md

@@ -521,6 +521,12 @@ paths:
  description: 'If set to true, return the historical metadata if seen in the delta log. This is for the streaming client to check if the table schema is still read compatible.'
  schema:
  type: boolean
+ - in: header
+ name: delta-sharing-capabilities
+ required: false
+ description: 'Delta Sharing Capabilities'
+ schema:
+ type: string
  responses:
  '400':
  $ref: "#/components/responses/400"
@@ -575,11 +581,14 @@ components:
  not Unary	Represents a logical not check. This op should have once child.
  The supported value types:
  ValueType	Description
- "bool"	Represents an Boolean type.
- "int"	Represents an Integer type.
- "long"	Represents a Long type.
- "string"	Represents a String type.
- "date"	Represents a Date type in "yyyy-mm-dd" format.
+ "bool" Represents an Boolean type.
+ "int" Represents an Integer type.
+ "long" Represents a Long type.
+ "string" Represents a String type.
+ "date" Represents a Date type in "yyyy-mm-dd" format.
+ "float" Represents a Float type.
+ "double" Represents a Double type.
+ "timestamp" Represents a timestamp in ISO8601 format, in the UTC timezone.
 
  ListShareResponse:
  type: object
@@ -758,124 +767,6 @@ components:
  type: string
  message:
  type: string
-
- # This is not used for the spec but comes handy for autogeneration
- TableMetadataResponseObject:
- type: object
- properties:
- protocol:
- # it refers to ./delta-sharing-protocol.md#protocol
- $ref: '#/components/schemas/ProtocolObject'
- metadata:
- # it refers to ./delta-sharing-protocol.md#metadata
- $ref: '#/components/schemas/MetadataObject'
-
- # This is not used for the spec but comes handy for autogeneration
- TableQueryResponseObject:
- type: object
- properties:
- protocol:
- # it refers to ./delta-sharing-protocol.md#protocol
- $ref: '#/components/schemas/ProtocolObject'
- metadata:
- # it refers to ./delta-sharing-protocol.md#metadata
- $ref: '#/components/schemas/MetadataObject'
- files:
- type: array
- items:
- # it refers to ./delta-sharing-protocol.md#file
- $ref: '#/components/schemas/FileObject'
- FileObject:
- type: object
- properties:
- file:
- type: object
- properties:
- url:
- type: string
- id:
- type: string
- partitionValues:
- type: object
- additionalProperties:
- type:
- string
- size:
- type: integer
- format: int64
- stats:
- type: string
- version:
- type: integer
- format: int64
- timestamp:
- type: integer
- format: int64
- expirationTimestamp:
- type: integer
- format: int64
- required:
- - url
- - id
- - partitionValues
- - size
- ProtocolObject:
- type: object
- properties:
- protocol:
- type: object
- properties:
- minReaderVersion:
- type: integer
- format: int32
- FormatObject:
- type: object
- properties:
- provider:
- type: string
- required:
- - provider
-
- MetadataObject:
- type: object
- properties:
- metaData:
- type: object
- properties:
- id:
- type: string
- name:
- type: string
- description:
- type: string
- format:
- $ref: '#/components/schemas/FormatObject'
- schemaString:
- type: string
- partitionColumns:
- type: array
- items:
- type: string
- configuration:
- type: object
- additionalProperties:
- type:
- string
- version:
- type: integer
- format: int64
- size:
- type: integer
- format: int64
- numFiles:
- type: integer
- format: int64
- required:
- - id
- - format
- - schemaString
- - partitionColumns
-
  responses:
  "400":
  description: The request is malformed

protocol/delta-sharing-protocol-api.yml

@@ -1,7 +1,13 @@
 package io.whitefox.api.deltasharing;
 
+import io.whitefox.api.deltasharing.model.v1.TableMetadataResponse;
+import io.whitefox.api.deltasharing.model.v1.TableQueryResponse;
 import io.whitefox.api.deltasharing.model.v1.generated.*;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetFile;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
 import io.whitefox.api.server.CommonMappers;
+import io.whitefox.api.server.WhitefoxMappers;
 import io.whitefox.core.*;
 import io.whitefox.core.Schema;
 import io.whitefox.core.Share;
@@ -53,46 +59,46 @@ public static ReadTableRequest api2ReadTableRequest(QueryRequest request) {
  }
  }
 
- public static TableQueryResponseObject readTableResult2api(ReadTableResult readTableResult) {
- return new TableQueryResponseObject()
- .metadata(metadata2Api(readTableResult.metadata()))
- .protocol(protocol2Api(readTableResult.protocol()))
- .files(readTableResult.files().stream()
- .map(DeltaMappers::file2Api)
- .collect(Collectors.toList()));
+ public static TableQueryResponse readTableResult2api(ReadTableResult readTableResult) {
+ return new TableQueryResponse(
+ protocol2Api(readTableResult.protocol()),
+ metadata2Api(readTableResult.metadata()),
+ readTableResult.files().stream().map(DeltaMappers::file2Api).collect(Collectors.toList()));
  }
 
- private static MetadataObject metadata2Api(Metadata metadata) {
- return new MetadataObject()
- .metaData(new MetadataObjectMetaData()
+ private static ParquetMetadata metadata2Api(Metadata metadata) {
+ return ParquetMetadata.builder()
+ .metadata(ParquetMetadata.Metadata.builder()
  .id(metadata.id())
- .name(metadata.name().orElse(null))
- .description(metadata.description().orElse(null))
- .format(new FormatObject().provider(metadata.format().provider()))
+ .name(metadata.name())
+ .description(metadata.description())
+ .format(WhitefoxMappers.format2api(metadata.format()))
  .schemaString(metadata.tableSchema().structType().toJson())
  .partitionColumns(metadata.partitionColumns())
- ._configuration(metadata.configuration())
- .version(metadata.version().orElse(null))
- .numFiles(metadata.numFiles().orElse(null)));
+ .configuration(Optional.of(metadata.configuration()))
+ .version(metadata.version())
+ .numFiles(metadata.numFiles())
+ .build())
+ .build();
  }
 
- private static ProtocolObject protocol2Api(Protocol protocol) {
- return new ProtocolObject()
- .protocol(new ProtocolObjectProtocol()
- .minReaderVersion(protocol.minReaderVersion().orElse(1)));
+ private static ParquetProtocol protocol2Api(Protocol protocol) {
+ return ParquetProtocol.ofMinReaderVersion(protocol.minReaderVersion().orElse(1));
  }
 
- private static FileObject file2Api(TableFile f) {
- return new FileObject()
- ._file(new FileObjectFile()
+ private static ParquetFile file2Api(TableFile f) {
+ return ParquetFile.builder()
+ .file(ParquetFile.File.builder()
  .id(f.id())
  .url(f.url())
  .partitionValues(f.partitionValues())
  .size(f.size())
- .stats(f.stats().orElse(null))
- .version(f.version().orElse(null))
- .timestamp(f.timestamp().orElse(null))
- .expirationTimestamp(f.expirationTimestamp()));
+ .stats(f.stats())
+ .version(f.version())
+ .timestamp(f.timestamp())
+ .expirationTimestamp(Optional.of(f.expirationTimestamp()))
+ .build())
+ .build();
  }
 
  public static TableReferenceAndReadRequest api2TableReferenceAndReadRequest(
@@ -127,9 +133,9 @@ public static Map<String, String> toHeaderCapabilitiesMap(String headerCapabilit
  .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
  }
 
- public static TableMetadataResponseObject toTableResponseMetadata(Metadata m) {
- return new TableMetadataResponseObject()
- .protocol(new ProtocolObject().protocol(new ProtocolObjectProtocol().minReaderVersion(1)))
- .metadata(metadata2Api(m));
+ public static TableMetadataResponse toTableResponseMetadata(Metadata m) {
+ return new TableMetadataResponse(
+ ParquetProtocol.ofMinReaderVersion(1), // smell
+ metadata2Api(m));
  }
 }

server/app/src/main/java/io/whitefox/api/deltasharing/DeltaMappers.java

@@ -0,0 +1,26 @@
+package io.whitefox.api.deltasharing.model.v1;
+
+import com.fasterxml.jackson.annotation.JsonCreator;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import lombok.EqualsAndHashCode;
+
+@EqualsAndHashCode
+public class Format {
+ private static final String PARQUET = "parquet";
+
+ @JsonProperty
+ public String provider() {
+ return PARQUET;
+ }
+
+ public Format() {
+ this(PARQUET);
+ }
+
+ @JsonCreator
+ private Format(@JsonProperty("provider") String provider) {
+ if (!"parquet".equalsIgnoreCase(provider)) {
+ throw new IllegalArgumentException("Provider must be " + PARQUET);
+ }
+ }
+}

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/Format.java

@@ -0,0 +1,13 @@
+package io.whitefox.api.deltasharing.model.v1;
+
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
+import lombok.NonNull;
+import lombok.Value;
+
+@Value
+public class TableMetadataResponse {
+ @NonNull ParquetProtocol protocol;
+
+ @NonNull ParquetMetadata metadata;
+}

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/TableMetadataResponse.java

@@ -0,0 +1,17 @@
+package io.whitefox.api.deltasharing.model.v1;
+
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetFile;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetMetadata;
+import io.whitefox.api.deltasharing.model.v1.parquet.ParquetProtocol;
+import java.util.List;
+import lombok.NonNull;
+import lombok.Value;
+
+@Value
+public class TableQueryResponse {
+ @NonNull ParquetProtocol protocol;
+
+ @NonNull ParquetMetadata metadata;
+
+ @NonNull List<ParquetFile> files;
+}

server/app/src/main/java/io/whitefox/api/deltasharing/model/v1/TableQueryResponse.java

@@ -0,0 +1,73 @@
+package io.whitefox.api.deltasharing.model.v1.delta;
+
+import com.fasterxml.jackson.annotation.JsonInclude;
+import com.fasterxml.jackson.annotation.JsonProperty;
+import com.fasterxml.jackson.databind.JsonNode;
+import java.util.Optional;
+import lombok.Builder;
+import lombok.NonNull;
+import lombok.Value;
+import lombok.experimental.SuperBuilder;
+import lombok.extern.jackson.Jacksonized;
+
+@Value
+@SuperBuilder
+@Jacksonized
+public class DeltaFile {
+
+ @JsonProperty
+ @NonNull File file;
+
+ @Value
+ @SuperBuilder
+ @Jacksonized
+ @JsonInclude(JsonInclude.Include.NON_ABSENT)
+ public static class File {
+
+ /**
+ * A unique string for the file in a table.
+ * The same file is guaranteed to have the same id across multiple requests.
+ * A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+ */
+ @JsonProperty
+ @NonNull String id;
+
+ /**
+ * A unique string for the deletion vector file in a table.
+ * The same deletion vector file is guaranteed to have the same id across multiple requests.
+ * A client may cache the file content and use this id as a key to decide whether to use the cached file content.
+ */
+ @JsonProperty
+ @Builder.Default
+ Optional<String> deletionVectorFileId = Optional.empty();
+
+ /**
+ * The table version of the file, returned when querying a table data with a version or timestamp parameter.
+ */
+ @JsonProperty
+ @Builder.Default
+ Optional<Long> version = Optional.empty();
+
+ /**
+ * The unix timestamp corresponding to the table version of the file, in milliseconds,
+ * returned when querying a table data with a version or timestamp parameter.
+ */
+ @JsonProperty
+ @Builder.Default
+ Optional<Long> timestamp = Optional.empty();
+
+ /**
+ * The unix timestamp corresponding to the expiration of the url, in milliseconds,
+ * returned when the server supports the feature.
+ */
+ @JsonProperty
+ @Builder.Default
+ Optional<Long> expirationTimestamp = Optional.empty();
+
+ /**
+ * Need to be parsed by a delta library as a delta single action, the path field is replaced by pr-signed url.
+ */
+ @JsonProperty
+ @NonNull JsonNode deltaSingleAction;
+ }
+}