add start of parsing schemas and artifacts into graph

This will generate the first schemas graph, and then prototype adding a single artifact (image reference and compatibility schema) onto it. Some design choices that I made are to store the mapping separate from the graph, that way we can always nuke them and start fresh without needing to recreate the graph. I also model a value (the leaf of the tree) as just another node, the idea being that we could extend the graph to include some new children. This sort of breaks the model that only the schemas should be nodes, but it is not so terrible an idea because in theory adding a value as a node on the fly could represent an updated cluster state (that is not present in the schema yet). THis might not be perfect but is a great start! I will next try to extract more of the artifacts and run this for a more reasonable image selection test case. Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

cmd/compspec/check/check.go

@@ -1,9 +1,11 @@
 package check
 
 import (
+"encoding/json"
 "fmt"
 "os"
 
+"github.com/supercontainers/compspec-go/pkg/graph"
 "github.com/supercontainers/compspec-go/pkg/oras"
 "github.com/supercontainers/compspec-go/pkg/types"
 "sigs.k8s.io/yaml"
@@ -47,17 +49,59 @@ func Run(manifestFile string, hostFields []string, mediaType string) error {
 }
 fmt.Println(manifestList)
 
+// Prepare a graph with our compspec schemas added
+g, err := graph.NewGraph()
+if err != nil {
+return err
+}
+
 // Load the compatibility specs into a lookup by image
 // This assumes we allow one image per compability spec, not sure
 // if there is a use case to have an image twice with two (sounds weird)
 lookup := map[string]types.CompatibilityRequest{}
 for _, item := range manifestList.Images {
+
 compspec, err := oras.LoadArtifact(item.Artifact, mediaType)
 if err != nil {
 fmt.Printf("warning, there was an issue loading the artifact for %s, skipping\n", item.Name)
 }
 lookup[item.Name] = compspec
+
+// Add schemas to the graph
+for _, schema := range compspec.Metadata.Schemas {
+fmt.Printf("Adding schema to graph %s\n", schema)
+err = g.AddSchema(schema)
+if err != nil {
+return err
+}
+}
+
+// When all schemas are added to a compatibility spec, we can walk graph to add metadata attributes
+// Each compspec has a list of compatibilities
+for _, compat := range compspec.Compatibilities {
+
+// If we don't have the root node, no go
+if !g.HasNode(compat.Name) {
+return fmt.Errorf("Schema root node %s is missing from the graph, missing from schemas.", compat.Name)
+}
+
+// Now add each attribute. Each attribute turns into a child node of the attribute
+// and if we are missing an attribute (meaning it isn't defined in the schema)
+// that is an error! item.Name is the container image to link to each node
+// in the traversal.
+for key, value := range compat.Attributes {
+err = g.AddAttribute(item.Name, compat.Name, key, value)
+}
+}
 }
+err = g.PrintMapping()
+if err != nil {
+return err
+}
+
+// Print the graph
+printme, _ := json.MarshalIndent(g.Graph, "", "\t")
+fmt.Println(string(printme))
 
 // TODO we will take this set of requests, load them into a graph,
 // and then query the graph based on user preferences (the host fields)

docs/usage.md

@@ -92,7 +92,7 @@ The idea here is that you can add custom metadata fields during your build, whic
  "metadata": {
  "name": "lammps-prototype",
  "schemas": {
- "archspec.io": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
+ "io.archspec": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
  "org.supercontainers": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
  }
  },
@@ -111,7 +111,7 @@ The idea here is that you can add custom metadata fields during your build, whic
  }
  },
  {
- "name": "archspec.io",
+ "name": "io.archspec",
  "version": "0.0.0",
  "attributes": {
  "cpu.model": "13th Gen Intel(R) Core(TM) i5-1335U",
@@ -145,6 +145,8 @@ provide the parameters about the expected runtime host to them.
 ./bin/compspec check -i ./examples/check-lammps/manifest.yaml
 ```
 
+TODO stopped here, need to do the whole check thing.
+
 <details>
 
 <summary>Example check output</summary>

examples/check-lammps/README.md

@@ -27,63 +27,56 @@ first, generating the artifact and [pushing to oras](https://oras.land/docs/how_
 mkdir -p ./specs
 
 # arguments are the hasGpu command and the path for the artifact (relative to PWD)
+# TODO it looks like the arch is coming from my host, this would need to be run at build time alongside
+# the machine it was built on. We'd also want to make sure it's documented this is the case
 cmd=". /etc/profile && /tmp/data/generate-artifact.sh no /tmp/data/specs/compspec-intel-mpi-rocky-9-amd64.json"
 docker run -v $PWD:/tmp/data -it ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64 /bin/bash -c "$cmd"
 
 # This generates ./specs/compspec-intel-mpi-rocky-9-amd64.json, let's push to a registry with oras
-oras push ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec-test --artifact-type application/org.supercontainers.compspec ./specs/compspec-intel-mpi-rocky-9-amd64.json:application/org.supercontainers.compspec
+oras push ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec --artifact-type application/org.supercontainers.compspec ./specs/compspec-intel-mpi-rocky-9-amd64.json:application/org.supercontainers.compspec
 ```
 
 Here is how we might see it:
 
 ```bash
-oras blob fetch --output - ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec@sha256:b68136afad3e4340f0dd4e09c5fea7faf12306cb4b0c1de616703b00d6ffef78
+oras blob fetch --output - ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64-compspec@sha256:376ea8d492aa8e8db312cecc34bbc729d18fc2b30e891deb2ffdffa38c7db3a5
 ```
 ```console
 {
  "version": "0.0.0",
  "kind": "CompatibilitySpec",
  "metadata": {
  "name": "lammps-prototype",
- "jsonSchema": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+ "schemas": {
+ "io.archspec": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
+ "org.supercontainers": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+ }
  },
  "compatibilities": [
  {
- "name": "org.supercontainers.mpi",
- "version": "0.0.0",
- "annotations": {
- "implementation": "intel-mpi",
- "version": "2021.8"
- }
- },
- {
- "name": "org.supercontainers.os",
- "version": "0.0.0",
- "annotations": {
- "name": "Rocky Linux 9.3 (Blue Onyx)",
- "release": "9.3",
- "vendor": "rocky",
- "version": "9.3"
- }
- },
- {
- "name": "org.supercontainers.hardware.gpu",
+ "name": "org.supercontainers",
  "version": "0.0.0",
- "annotations": {
- "available": "no"
+ "attributes": {
+ "hardware.gpu.available": "no",
+ "mpi.implementation": "intel-mpi",
+ "mpi.version": "2021.8",
+ "os.name": "Rocky Linux 9.3 (Blue Onyx)",
+ "os.release": "9.3",
+ "os.vendor": "rocky",
+ "os.version": "9.3"
  }
  },
  {
- "name": "io.archspec.cpu",
+ "name": "io.archspec",
  "version": "0.0.0",
- "annotations": {
- "model": "13th Gen Intel(R) Core(TM) i5-1335U",
- "target": "amd64",
- "vendor": "GenuineIntel"
+ "attributes": {
+ "cpu.model": "13th Gen Intel(R) Core(TM) i5-1335U",
+ "cpu.target": "amd64",
+ "cpu.vendor": "GenuineIntel"
  }
  }
  ]
 }
 ```
 
-This is great! Next we will capture the URIs of these together in a manifest and put into our compspec tool. TBA!
+This is great! Next we will capture the URIs of these together in a manifest and put into our compspec tool.

examples/check-lammps/generate-artifact.sh

@@ -5,7 +5,7 @@ hasGpu="${1:-no}"
 path="${2:-./compatibility-spec.json}"
 
 # Note that this is hard coded for amd64, for arm you would wantt o add -arm or ppc64le -ppc
-wget --quiet https://github.com/supercontainers/compspec-go/releases/download/1-26-2024-2/compspec
+wget --quiet https://github.com/supercontainers/compspec-go/releases/download/2-2-2024-1/compspec
 chmod +x compspec
 
 # Download the spec for our compatibility artifact

examples/check-lammps/specs/compspec-intel-mpi-rocky-9-amd64.json

@@ -3,41 +3,32 @@
  "kind": "CompatibilitySpec",
  "metadata": {
  "name": "lammps-prototype",
- "jsonSchema": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+ "schemas": {
+ "io.archspec": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
+ "org.supercontainers": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+ }
  },
  "compatibilities": [
  {
- "name": "org.supercontainers.mpi",
- "version": "0.0.0",
- "annotations": {
- "implementation": "intel-mpi",
- "version": "2021.8"
- }
- },
- {
- "name": "org.supercontainers.os",
- "version": "0.0.0",
- "annotations": {
- "name": "Rocky Linux 9.3 (Blue Onyx)",
- "release": "9.3",
- "vendor": "rocky",
- "version": "9.3"
- }
- },
- {
- "name": "org.supercontainers.hardware.gpu",
+ "name": "org.supercontainers",
  "version": "0.0.0",
- "annotations": {
- "available": "no"
+ "attributes": {
+ "hardware.gpu.available": "no",
+ "mpi.implementation": "intel-mpi",
+ "mpi.version": "2021.8",
+ "os.name": "Rocky Linux 9.3 (Blue Onyx)",
+ "os.release": "9.3",
+ "os.vendor": "rocky",
+ "os.version": "9.3"
  }
  },
  {
- "name": "io.archspec.cpu",
+ "name": "io.archspec",
  "version": "0.0.0",
- "annotations": {
- "model": "13th Gen Intel(R) Core(TM) i5-1335U",
- "target": "amd64",
- "vendor": "GenuineIntel"
+ "attributes": {
+ "cpu.model": "13th Gen Intel(R) Core(TM) i5-1335U",
+ "cpu.target": "amd64",
+ "cpu.vendor": "GenuineIntel"
  }
  }
  ]

examples/generated-compatibility-spec.json

@@ -4,7 +4,7 @@
  "metadata": {
  "name": "lammps-prototype",
  "schemas": {
- "archspec.io": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
+ "io.archspec": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
  "org.supercontainers": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
  }
  },
@@ -23,7 +23,7 @@
  }
  },
  {
- "name": "archspec.io",
+ "name": "io.archspec",
  "version": "0.0.0",
  "attributes": {
  "cpu.model": "13th Gen Intel(R) Core(TM) i5-1335U",

examples/lammps-experiment.yaml

@@ -10,7 +10,7 @@ metadata:
  # "Validate the namespaced attributes with these schemas"
  schemas:
  org.supercontainers: https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json
- archspec.io: https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json
+ io.archspec: https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json
 
 # These are not values, but mappings, from an extractor into the compspec we want
 compatibilities:
@@ -28,7 +28,7 @@ compatibilities:
 # Note that for now we are using the processor in index 0 to represent all
 # I'm not sure about cases where this set isn't homogeneous!
 # Since target is part of the container build, we will provide it
-- name: "archspec.io"
+- name: "io.archspec"
  version: "0.0.0"
  attributes:
  cpu.model: system.processor.0.model

go.mod

@@ -5,6 +5,7 @@ go 1.20
 require (
 github.com/akamensky/argparse v1.4.0
 github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8
+github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3
 github.com/jedib0t/go-pretty/v6 v6.5.3
 github.com/moby/moby v25.0.0+incompatible
 github.com/opencontainers/image-spec v1.1.0-rc5

go.sum

@@ -4,6 +4,8 @@ github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8 h1:SjZ2GvvOononHOpK
 github.com/c9s/goprocinfo v0.0.0-20210130143923-c95fcf8c64a8/go.mod h1:uEyr4WpAH4hio6LFriaPkL938XnrvLpNPmQHBdrmbIE=
 github.com/containerd/log v0.1.0 h1:TCJt7ioM2cr/tfR8GPbGf9/VRAX8D2B4PjzCpfX540I=
 github.com/containerd/log v0.1.0/go.mod h1:VRRf09a7mHDIRezVKTRCrOq78v577GXq3bSa3EhrzVo=
+github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3 h1:frJJfyARuHmF2eohDCyltBLE6tRJKvA1shuS2aWQaf8=
+github.com/converged-computing/jsongraph-go v0.0.0-20231221142916-249fef6889b3/go.mod h1:+DhVyLXGVfBsfta4185jd33jqa94inshCcdvsXK2Irk=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=

pkg/graph/edges.go

@@ -0,0 +1,10 @@
+package graph
+
+import (
+"github.com/converged-computing/jsongraph-go/jsongraph/v2/graph"
+)
+
+// Get an edge with a specific containment (typically "contains" or "in")
+func getEdge(source string, dest string, containment string) graph.Edge {
+return graph.Edge{Source: source, Target: dest, Relation: containment}
+}