adding support to cache entire graph

with repeated calls, we should not need to derive it from source every time. Signed-off-by: vsoch <vsoch@users.noreply.github.com>

Author: vsoch

cmd/compspec/compspec.go

@@ -46,12 +46,13 @@ func main() {
 matchFields := matchCmd.StringList("m", "match", &argparse.Options{Help: "One or more key value pairs to match"})
 manifestFile := matchCmd.String("i", "in", &argparse.Options{Required: true, Help: "Input manifest list yaml that contains pairs of images and artifacts"})
 printMapping := matchCmd.Flag("p", "print", &argparse.Options{Help: "Print mapping of images to attributes only."})
-printGraph := matchCmd.Flag("g", "print-graph", &argparse.Options{Help: "Print schema graph"})
+printGraph := matchCmd.Flag("", "print-graph", &argparse.Options{Help: "Print schema graph"})
 checkArtifacts := matchCmd.Flag("c", "check-artifacts", &argparse.Options{Help: "Check that all artifacts exist"})
 allowFailMatch := matchCmd.Flag("f", "allow-fail", &argparse.Options{Help: "Allow an artifact to be missing (and not included)"})
 randomize := matchCmd.Flag("r", "randomize", &argparse.Options{Help: "Shuffle match results in random order"})
 single := matchCmd.Flag("s", "single", &argparse.Options{Help: "Only return a single result"})
 cachePath := matchCmd.String("", "cache", &argparse.Options{Help: "A path to a cache for artifacts"})
+saveGraph := matchCmd.String("", "cache-graph", &argparse.Options{Help: "Load or use a cached graph"})
 
 // Create arguments
 options := createCmd.StringList("a", "append", &argparse.Options{Help: "Append one or more custom metadata fields to append"})
@@ -84,6 +85,7 @@ func main() {
 *matchFields,
 *mediaType,
 *cachePath,
+*saveGraph,
 *printMapping,
 *printGraph,
 *allowFailMatch,

cmd/compspec/match/match.go

@@ -38,6 +38,7 @@ func Run(
 hostFields []string,
 mediaType string,
 cachePath string,
+graphPath string,
 printMapping bool,
 printGraph bool,
 allowFail bool,
@@ -66,6 +67,17 @@ func Run(
 return err
 }
 
+// If we are given a graph path and it exists, load it
+// This won't fail if it does not exist, but will error
+// with another issue
+exists := false
+if graphPath != "" {
+exists, err = g.LoadGraph(graphPath)
+if err != nil {
+return err
+}
+}
+
 // Keep check of artifacts missing
 missing := 0
 
@@ -91,11 +103,13 @@ func Run(
 }
 lookup[item.Name] = compspec
 
-// Add schemas to the graph
-for _, schema := range compspec.Metadata.Schemas {
-err = g.AddSchema(schema)
-if err != nil {
-return err
+// Add schemas to the graph TODO, allow caching
+if !exists {
+for _, schema := range compspec.Metadata.Schemas {
+err = g.AddSchema(schema)
+if err != nil {
+return err
+}
 }
 }
 
@@ -118,6 +132,14 @@ func Run(
 }
 }
 
+// Save graph if given a file
+if graphPath != "" {
+err = g.SaveGraph(graphPath)
+if err != nil {
+return err
+}
+}
+
 if checkArtifacts {
 fmt.Printf("Checking artifacts complete. There were %d artifacts missing.\n", missing)
 return nil

docs/usage.md

@@ -243,7 +243,19 @@ mkdir -p ./cache
 ./bin/compspec match -i ./examples/check-lammps/manifest.yaml --cache ./cache
 ```
 
+You can also save the graph to file:
 
+```bash
+./bin/compspec match -i ./examples/check-lammps/manifest.yaml --cache ./cache --cache-graph ./cache/lammps-experiment.json
+```
+
+You can then use that cached graph later (NOTE this is at your own discretion knowing the schemas needed).
+
+```bash
+./bin/compspec match -i ./examples/check-lammps/manifest.yaml --cache ./cache --cache-graph ./cache/lammps-experiment.json
+```
+
+Likely we will make tools to visualize it that can just show JGF!
 
 ### Match to print Metadata attributes
 
@@ -604,4 +616,4 @@ The ordering of your list is honored.
 ## Developer
 
 Note that there is a [developer environment](.devcontainer) that provides a consistent version of Go, etc.
-However, it won't work with all extractors. Note that for any command that uses a plugin (e.g., `extract` and `check`)
+However, it won't work with all extractors. Note that for any command that uses a plugin (e.g., `extract` and `check`)

pkg/graph/graph.go

@@ -1,7 +1,9 @@
 package graph
 
 import (
+"encoding/json"
 "fmt"
+"os"
 "strings"
 
 "github.com/converged-computing/jsongraph-go/jsongraph/v2/graph"
@@ -222,6 +224,61 @@ func (c *CompatibilityGraph) Match(fields []string) ([]string, error) {
 return matches.List(), nil
 }
 
+// Load graph from a cached file.
+// This assumes you know what you are doing, meaning
+// the schemas are not changing
+func (c *CompatibilityGraph) LoadGraph(path string) (bool, error) {
+exists, err := utils.PathExists(path)
+
+// Some error, do not continue!
+if err != nil {
+return exists, err
+}
+// No error, but doesn't exist (will still load)
+if !exists {
+return exists, nil
+}
+
+// If we get here, load in!
+fd, err := os.Open(path)
+if err != nil {
+return exists, err
+}
+graph := graph.Graph{}
+jsonParser := json.NewDecoder(fd)
+err = jsonParser.Decode(&graph)
+if err != nil {
+return exists, err
+}
+
+// Go through the edges and add the root nodes
+c.Graph = graph
+return true, nil
+}
+
+// Save graph to a cached file
+func (c *CompatibilityGraph) SaveGraph(path string) error {
+exists, err := utils.PathExists(path)
+if err != nil {
+return err
+}
+// Don't overwrite if exists
+if exists {
+fmt.Printf("Graph %s already exists, will not overwrite\n", path)
+return nil
+}
+content, err := json.MarshalIndent(c.Graph, "", " ")
+if err != nil {
+return err
+}
+fmt.Printf("Saving graph to %s\n", path)
+err = os.WriteFile(path, content, 0644)
+if err != nil {
+return err
+}
+return nil
+}
+
 // AddSchema by url to the graph
 func (c *CompatibilityGraph) AddSchema(url string) error {