compress symbol graph uploads

Sources/UnidocAPI/Building/Unidoc.BuildRoute.swift

@@ -3,12 +3,10 @@ extension Unidoc
  @frozen public
  enum BuildRoute:String, Sendable
  {
+ /// Route for uploading symbol graphs.
+ case artifact
  /// Route for uploading build status and logs.
  case report
- /// Route for uploading labeled symbol graphs.
- case labeled
- /// Route for uploading unlabeled symbol graphs.
- case labeling
  }
 }
 extension Unidoc.BuildRoute:CustomStringConvertible

Sources/UnidocAPI/Building/Unidoc.LinkerAction.swift

@@ -5,8 +5,7 @@ extension Unidoc
  @frozen public
  enum LinkerAction:Int32, Equatable, Sendable
  {
- case uplinkInitial = 0
- case uplinkRefresh
+ case uplink = 1
  case unlink
  case delete
  }
@@ -18,10 +17,9 @@ extension Unidoc.LinkerAction:CustomStringConvertible
  {
  switch self
  {
- case .uplinkInitial: "UPLINK_INITIAL"
- case .uplinkRefresh: "UPLINK_REFRESH"
- case .unlink: "UNLINK"
- case .delete: "DELETE"
+ case .uplink: "UPLINK"
+ case .unlink: "UNLINK"
+ case .delete: "DELETE"
  }
  }
 }
@@ -32,11 +30,10 @@ extension Unidoc.LinkerAction:LosslessStringConvertible
  {
  switch description
  {
- case "UPLINK_INITIAL": self = .uplinkInitial
- case "UPLINK_REFRESH": self = .uplinkRefresh
- case "UNLINK": self = .unlink
- case "DELETE": self = .delete
- default: return nil
+ case "UPLINK": self = .uplink
+ case "UNLINK": self = .unlink
+ case "DELETE": self = .delete
+ default: return nil
  }
  }
 }

Sources/UnidocClient/Unidoc.BuildPayload (ext).swift

@@ -0,0 +1,30 @@
+import BSON
+import LZ77
+import SymbolGraphs
+
+extension Unidoc.BuildPayload
+{
+ init(metadata:SymbolGraphMetadata, graph:borrowing SymbolGraph, level:Int = 12)
+ {
+ let document:BSON.Document = .init(encoding: graph)
+ let size:Int64 = .init(document.bytes.count)
+
+ print("Compressing symbol graph (\(size / 1_000) KB)...")
+
+ var deflator:LZ77.Deflator
+
+ deflator = .init(format: .zlib, level: level, hint: 128 << 10)
+ deflator.push(document.bytes, last: true)
+
+ var bytes:[UInt8] = []
+
+ while let part:[UInt8] = deflator.pull()
+ {
+ bytes += part
+ }
+
+ print("Compressed symbol graph (\(bytes.count / 1_000) KB)")
+
+ self.init(metadata: metadata, zlib: bytes[...], size: size)
+ }
+}

Sources/UnidocClient/Unidoc.Client.Connection.swift

@@ -70,9 +70,9 @@ extension Unidoc.Client<HTTP.Client2>.Connection
 }
 extension Unidoc.Client<HTTP.Client1>.Connection
 {
- func upload(_ unlabeled:SymbolGraphObject<Void>) async throws
+ func upload(_ artifact:Unidoc.BuildArtifact) async throws
  {
- let bson:BSON.Document = .init(encoding: unlabeled)
+ let bson:BSON.Document = .init(encoding: artifact)
 
  print("Uploading local symbol graph...")
 
@@ -82,7 +82,7 @@ extension Unidoc.Client<HTTP.Client1>.Connection
  let body:ByteBuffer = self.http.buffer(bytes: bson.bytes)
 
  let request:HTTP.Client1.Request = .init(method: .PUT,
- path: "/builder/\(Unidoc.BuildRoute.labeling)",
+ path: "/builder/\(Unidoc.BuildRoute.artifact)",
  head: [
  "content-type": "\(type)",
  "content-length": "\(body.readableBytes)",
@@ -110,30 +110,6 @@ extension Unidoc.Client<HTTP.Client1>.Connection
 }
 extension Unidoc.Client<HTTP.Client2>.Connection
 {
- func upload(_ unlabeled:SymbolGraphObject<Void>) async throws
- {
- let bson:BSON.Document = .init(encoding: unlabeled)
-
- print("Uploading unlabeled symbol graph...")
-
- do
- {
- let _:Never = try await self.put(bson: bson,
- to: "/builder/\(Unidoc.BuildRoute.labeling)",
- timeout: .seconds(60))
- }
- catch is HTTP.NonError
- {
- print("Successfully uploaded unlabeled symbol graph!")
- }
- catch let error
- {
- print("Error: failed to upload unlabeled symbol graph!")
- print("Error: \(error)")
- throw error
- }
- }
-
  func upload(_ artifact:Unidoc.BuildArtifact) async throws
  {
  let bson:BSON.Document = .init(encoding: artifact)
@@ -143,7 +119,7 @@ extension Unidoc.Client<HTTP.Client2>.Connection
  do
  {
  let _:Never = try await self.put(bson: bson,
- to: "/builder/\(Unidoc.BuildRoute.labeled)",
+ to: "/builder/\(Unidoc.BuildRoute.artifact)",
  timeout: .seconds(60))
  }
  catch is HTTP.NonError

Sources/UnidocClient/Unidoc.Client.swift

@@ -130,7 +130,6 @@ extension Unidoc.Client<HTTP.Client2>
  @discardableResult public
  func buildAndUpload(
  labels:Unidoc.BuildLabels,
- action:Unidoc.LinkerAction,
  remove:Bool = false,
  with toolchain:Unidoc.Toolchain,
  cache:FilePath? = nil) async throws -> Bool
@@ -248,10 +247,7 @@ extension Unidoc.Client<HTTP.Client2>
  {
  let object:SymbolGraphObject<Void> = try .init(buffer: try docs.read())
 
- artifact.outcome = .success(.init(id: labels.coordinate,
- metadata: object.metadata,
- inline: object.graph,
- action: action))
+ artifact.outcome = .success(.init(metadata: object.metadata, graph: object.graph))
  }
 
  // Attach build logs.
@@ -289,7 +285,10 @@ extension Unidoc.Client<HTTP.Client2>
  type: type,
  with: toolchain)
 
- try await self.connect { try await $0.upload(object) }
+ let artifact:Unidoc.BuildArtifact = .init(edition: nil,
+ outcome: .success(.init(metadata: object.metadata, graph: object.graph)))
+
+ try await self.connect { try await $0.upload(artifact) }
 
  // The final URL path component might have different casing than the directory name.
  print("""
@@ -311,14 +310,17 @@ extension Unidoc.Client<HTTP.Client1>
  type: type,
  with: toolchain)
 
- try await self.connect { try await $0.upload(object) }
+ let artifact:Unidoc.BuildArtifact = .init(edition: nil,
+ outcome: .success(.init(metadata: object.metadata, graph: object.graph)))
+
+ try await self.connect { try await $0.upload(artifact) }
 
  print("""
  View the generated documentation at:
  http://\(self.http.remote):\(self.port)/tags/\(object.metadata.package.id)
  """)
  }
-} 
+}
 extension Unidoc.Client
 {
  /// Name is case-sensitive, so it is not modeled as a ``Symbol.Package``.
@@ -353,7 +355,7 @@ extension Unidoc.Client
  arguments.append("--sdk")
  arguments.append("\(sdk)")
  }
- if let local:FilePath.Directory 
+ if let local:FilePath.Directory
  {
  arguments.append("--project-path")
  arguments.append("\(local)")

Sources/UnidocDB/Unidoc.DB.swift

@@ -402,11 +402,16 @@ extension Unidoc.DB
  linking docs:SymbolGraphObject<Void>)
  async throws -> (Unidoc.UploadStatus, Unidoc.UplinkStatus)
  {
- var snapshot:Unidoc.Snapshot
- let realm:Unidoc.Realm?
+ let package:Unidoc.PackageMetadata
+ let edition:Unidoc.EditionMetadata
+
+ (package, edition) = try await self.label(docs: docs.metadata)
 
  // Don’t queue for uplink, since we’re going to do that synchronously.
- (snapshot, realm) = try await self.label(documentation: docs, action: nil)
+ var snapshot:Unidoc.Snapshot = .init(id: edition.id,
+ metadata: docs.metadata,
+ inline: docs.graph,
+ action: nil)
 
  enum NoLoader:Unidoc.GraphLoader
  {
@@ -419,7 +424,7 @@ extension Unidoc.DB
  let linked:Unidoc.Mesh = try await self.link(&snapshot,
  symbol: docs.metadata.package.id,
  loader: nil as NoLoader?,
- realm: realm)
+ realm: package.realm)
  /// This is here to workaround a Swift compiler bug.
  let volume:Symbol.Volume = linked.volume
 
@@ -441,42 +446,43 @@ extension Unidoc.DB
  public
  func store(docs:consuming SymbolGraphObject<Void>) async throws -> Unidoc.UploadStatus
  {
- let (snapshot, _):(Unidoc.Snapshot, Unidoc.Realm?) = try await self.label(
- documentation: docs,
- action: .uplinkInitial)
+ let (_, edition):(_, Unidoc.EditionMetadata) = try await self.label(docs: docs.metadata)
+
+ let snapshot:Unidoc.Snapshot = .init(id: edition.id,
+ metadata: docs.metadata,
+ inline: docs.graph,
+ action: .uplink)
 
  return try await self.snapshots.store(snapshot: snapshot)
  }
 
  public
- func label(
- documentation:consuming SymbolGraphObject<Void>,
- action:Unidoc.LinkerAction?) async throws ->
- (
- snapshot:Unidoc.Snapshot,
- realm:Unidoc.Realm?
- )
+ func label(docs:SymbolGraphMetadata) async throws ->
+ (
+ package:Unidoc.PackageMetadata,
+ edition:Unidoc.EditionMetadata
+ )
  {
  let (package, _):(Unidoc.PackageMetadata, Bool) = try await self.index(
- package: documentation.metadata.package.id,
+ package: docs.package.id,
  repo: nil)
 
  let edition:Unidoc.EditionMetadata
- if let commit:SymbolGraphMetadata.Commit = documentation.metadata.commit
+ if let commit:SymbolGraphMetadata.Commit = docs.commit
  {
  (edition, _) = try await self.index(
  package: package.id,
- version: documentation.metadata.package.name.version(tag: commit.name),
+ version: docs.package.name.version(tag: commit.name),
  name: commit.name,
  sha1: commit.sha1)
  }
  else if
- case .swift = documentation.metadata.package.name,
- case nil = documentation.metadata.swift.nightly
+ case .swift = docs.package.name,
+ case nil = docs.swift.nightly
  {
  (edition, _) = try await self.index(
  package: package.id,
- version: .release(documentation.metadata.swift.version),
+ version: .release(docs.swift.version),
  name: "__Xcode",
  sha1: nil)
  }
@@ -495,12 +501,7 @@ extension Unidoc.DB
  try await self.editions.upsert(edition)
  }
 
- let snapshot:Unidoc.Snapshot = .init(id: edition.id,
- metadata: documentation.metadata,
- inline: documentation.graph,
- action: action)
-
- return (snapshot, package.realm)
+ return (package, edition)
  }
 }
 extension Unidoc.DB

Sources/UnidocLinkerPlugin/Unidoc.GraphLinker.swift

@@ -58,7 +58,7 @@ extension Unidoc.GraphLinker
  action:
  switch operation.action
  {
- case .uplinkInitial, .uplinkRefresh:
+ case .uplink:
  guard
  let status:Unidoc.UplinkStatus = try await context.db.uplink(operation.edition,
  from: graphs)

Sources/UnidocRecords/Building/Unidoc.BuildArtifact.swift

@@ -7,9 +7,9 @@ extension Unidoc
  struct BuildArtifact:Sendable
  {
  public
- let edition:Edition
+ let edition:Edition?
  public
- var outcome:Result<Snapshot, BuildFailure>
+ var outcome:Result<BuildPayload, BuildFailure>
  public
  var seconds:Int64
  public
@@ -18,8 +18,8 @@ extension Unidoc
  var logsAreSecret:Bool
 
  @inlinable public
- init(edition:Edition,
- outcome:Result<Snapshot, BuildFailure>,
+ init(edition:Edition?,
+ outcome:Result<BuildPayload, BuildFailure>,
  seconds:Int64 = 0,
  logs:[BuildLog] = [],
  logsAreSecret:Bool = false)
@@ -80,19 +80,19 @@ extension Unidoc.BuildArtifact:BSONDocumentDecodable
  public
  init(bson:BSON.DocumentDecoder<CodingKey>) throws
  {
- let outcome:Result<Unidoc.Snapshot, Unidoc.BuildFailure>
+ let outcome:Result<Unidoc.BuildPayload, Unidoc.BuildFailure>
 
- if let snapshot:Unidoc.Snapshot = try bson[.payload]?.decode()
+ if let payload:Unidoc.BuildPayload = try bson[.payload]?.decode()
  {
- outcome = .success(snapshot)
+ outcome = .success(payload)
  }
  else
  {
  outcome = .failure(try bson[.failure].decode())
  }
 
  self.init(
- edition: try bson[.edition].decode(),
+ edition: try bson[.edition]?.decode(),
  outcome: outcome,
  seconds: try bson[.seconds].decode(),
  logs: try bson[.logs]?.decode() ?? [],