Merge pull request #14 from supercontainers/add-randomize

add support for cache and randomize

Author: vsoch

cmd/compspec/compspec.go

@@ -49,6 +49,9 @@ func main() {
 printGraph := matchCmd.Flag("g", "print-graph", &argparse.Options{Help: "Print schema graph"})
 checkArtifacts := matchCmd.Flag("c", "check-artifacts", &argparse.Options{Help: "Check that all artifacts exist"})
 allowFailMatch := matchCmd.Flag("f", "allow-fail", &argparse.Options{Help: "Allow an artifact to be missing (and not included)"})
+randomize := matchCmd.Flag("r", "randomize", &argparse.Options{Help: "Shuffle match results in random order"})
+single := matchCmd.Flag("s", "single", &argparse.Options{Help: "Only return a single result"})
+cachePath := matchCmd.String("", "cache", &argparse.Options{Help: "A path to a cache for artifacts"})
 
 // Create arguments
 options := createCmd.StringList("a", "append", &argparse.Options{Help: "Append one or more custom metadata fields to append"})
@@ -76,11 +79,21 @@ func main() {
 log.Fatal(err.Error())
 }
 } else if matchCmd.Happened() {
-err := match.Run(*manifestFile, *matchFields, *mediaType, *printMapping, *printGraph, *allowFailMatch, *checkArtifacts)
+err := match.Run(
+*manifestFile,
+*matchFields,
+*mediaType,
+*cachePath,
+*printMapping,
+*printGraph,
+*allowFailMatch,
+*checkArtifacts,
+*randomize,
+*single,
+)
 if err != nil {
 log.Fatal(err.Error())
 }
-
 } else if listCmd.Happened() {
 err := list.Run(*pluginNames)
 if err != nil {

cmd/compspec/match/match.go

@@ -8,6 +8,7 @@ import (
 "github.com/supercontainers/compspec-go/pkg/graph"
 "github.com/supercontainers/compspec-go/pkg/oras"
 "github.com/supercontainers/compspec-go/pkg/types"
+"github.com/supercontainers/compspec-go/pkg/utils"
 "sigs.k8s.io/yaml"
 )
 
@@ -36,10 +37,13 @@ func Run(
 manifestFile string,
 hostFields []string,
 mediaType string,
+cachePath string,
 printMapping bool,
 printGraph bool,
 allowFail bool,
 checkArtifacts bool,
+randomize bool,
+single bool,
 ) error {
 
 // Default media type if one not provided
@@ -71,7 +75,7 @@ func Run(
 lookup := map[string]types.CompatibilityRequest{}
 for _, item := range manifestList.Images {
 
-compspec, err := oras.LoadArtifact(item.Artifact, mediaType)
+compspec, err := oras.LoadArtifact(item.Artifact, mediaType, cachePath)
 if err != nil {
 fmt.Printf("warning, there was an issue loading the artifact for %s: %s, skipping\n", item.Name, err)
 
@@ -142,11 +146,23 @@ func Run(
 }
 if len(matches) == 0 {
 fmt.Println("There was no match. Try changig your constaints.")
-} else {
-fmt.Println(" --- Found matches ---")
-for _, match := range matches {
-fmt.Println(match)
-}
+return nil
 }
+
+// Do we want to randomize (randomly shuffle the list)?
+if randomize {
+matches = utils.RandomSort(matches)
+}
+
+// Do we want just one match?
+if single && len(matches) > 0 {
+matches = []string{matches[0]}
+}
+
+fmt.Println(" --- Found matches ---")
+for _, match := range matches {
+fmt.Println(match)
+}
+
 return nil
 }

docs/usage.md

@@ -204,6 +204,46 @@ ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-8-amd64
 
 That is very simple, but should serve our purposes for now.
 
+### Match with randomize or single
+
+You can choose to shuffle the results:
+
+```bash
+$ ./bin/compspec match -i ./examples/check-lammps/manifest.yaml --randomize
+```
+```console
+Schema io.archspec is being added to the graph
+Schema org.supercontainers is being added to the graph
+No field criteria provided, all images are matches.
+ --- Found matches ---
+ghcr.io/rse-ops/lammps-matrix:openmpi-ubuntu-gpu-20.04
+ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64
+ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-8-amd64
+ghcr.io/rse-ops/lammps-matrix:openmpi-ubuntu-gpu-22.04
+```
+Or do the same, but ask for only one result (note we might) change this to N results depending on use cases:
+
+```bash
+$ ./bin/compspec match -i ./examples/check-lammps/manifest.yaml --randomize --single
+```
+```console
+Schema io.archspec is being added to the graph
+Schema org.supercontainers is being added to the graph
+No field criteria provided, all images are matches.
+ --- Found matches ---
+ghcr.io/rse-ops/lammps-matrix:intel-mpi-rocky-9-amd64
+```
+
+### Match and use cache
+
+If you intend to run a match request many times (using repeated images) it's good practice to use a cache. This means you'll look in the cache before asking a registry, and save after if it doesn't exist. The directory must exist.
+
+```bash
+mkdir -p ./cache
+./bin/compspec match -i ./examples/check-lammps/manifest.yaml --cache ./cache
+```
+
+
 
 ### Match to print Metadata attributes
 

pkg/oras/oras.go

@@ -4,21 +4,117 @@ import (
 "bytes"
 "context"
 "encoding/json"
+"fmt"
 "io"
+"os"
+"path/filepath"
 "strings"
 
 oci "github.com/opencontainers/image-spec/specs-go/v1"
 "github.com/supercontainers/compspec-go/pkg/types"
+"github.com/supercontainers/compspec-go/pkg/utils"
 "oras.land/oras-go/v2/content"
 "oras.land/oras-go/v2/registry/remote"
 "sigs.k8s.io/yaml"
 )
 
+// toFilename converts the uri of an image to a filename
+func toFilename(uri string) string {
+for _, repl := range []string{"/", ":"} {
+uri = strings.ReplaceAll(uri, repl, "-")
+}
+return fmt.Sprintf("%s.json", uri)
+}
+
+// LoadFromCache loads the compatibility request from cache
+func LoadFromCache(uri, cache string) (types.CompatibilityRequest, error) {
+var request types.CompatibilityRequest
+var err error
+cachePath := filepath.Join(cache, toFilename(uri))
+exists, err := utils.PathExists(cachePath)
+if err != nil {
+return request, err
+}
+if exists {
+fd, err := os.Open(cachePath)
+b, err := io.ReadAll(fd)
+if err != nil {
+return request, err
+}
+defer fd.Close()
+
+err = json.Unmarshal(b, &request)
+}
+return request, err
+}
+
 // Oras will provide an interface to retrieve an artifact, specifically
 // a compatibillity spec artifact media type
 // LoadArtifact retrieves the artifact from the url string
 // and returns based on the media type
-func LoadArtifact(uri string, mediaType string) (types.CompatibilityRequest, error) {
+func LoadArtifact(
+uri string,
+mediaType string,
+cache string,
+) (types.CompatibilityRequest, error) {
+
+request := types.CompatibilityRequest{}
+var err error
+
+// If cache is desired and we have the artifact
+if cache != "" {
+
+// Must exist
+exists, err := utils.PathExists(cache)
+if err != nil {
+return request, err
+}
+if !exists {
+return request, fmt.Errorf("Cache path %s does not exist", cache)
+}
+request, err := LoadFromCache(uri, cache)
+if err != nil {
+return request, err
+}
+}
+
+// If we didn't get matches, load from registry
+if request.Kind == "" {
+request, err = LoadFromRegistry(uri, mediaType)
+
+// If we loaded it and have a cache, save to cache
+if cache != "" {
+err = SaveToCache(request, uri, cache)
+}
+}
+return request, err
+}
+
+// Save to cache
+func SaveToCache(request types.CompatibilityRequest, uri, cache string) error {
+cachePath := filepath.Join(cache, toFilename(uri))
+exists, err := utils.PathExists(cachePath)
+if err != nil {
+return err
+}
+
+// Don't overwrite
+if exists {
+return nil
+}
+content, err := json.Marshal(request)
+if err != nil {
+return err
+}
+err = os.WriteFile(cachePath, content, 0644)
+if err != nil {
+return err
+}
+return nil
+}
+
+// Load the artifact from a registry
+func LoadFromRegistry(uri, mediaType string) (types.CompatibilityRequest, error) {
 request := types.CompatibilityRequest{}
 ctx := context.Background()
 repo, err := remote.NewRepository(uri)

pkg/utils/utils.go

@@ -4,18 +4,19 @@ import (
 "bufio"
 "errors"
 "fmt"
+"math/rand"
 "os"
 "strings"
 )
 
-// FileExists determines if a file exists
-func FileExists(path string) (bool, error) {
+// PathExists determines if a path exists
+func PathExists(path string) (bool, error) {
 _, err := os.Stat(path)
 if err != nil {
 if errors.Is(err, os.ErrNotExist) {
 return false, nil
 }
-return true, fmt.Errorf("warning: file exists but another error happened (debug): %s", err)
+return true, fmt.Errorf("warning: exists but another error happened (debug): %s", err)
 }
 return true, nil
 }
@@ -107,3 +108,12 @@ func ParseConfigFile(path, comment, delim string) (map[string]string, error) {
 }
 return data, nil
 }
+
+// RandomSort an array of strings, in place
+func RandomSort(items []string) []string {
+for i := range items {
+j := rand.Intn(i + 1)
+items[i], items[j] = items[j], items[i]
+}
+return items
+}

plugins/extractors/system/arch.go

@@ -23,7 +23,7 @@ func getOsArch() (string, error) {
 
 // Detect OS architecture based on presence of this file
 for arch, path := range linkerPaths {
-exists, err := utils.FileExists(path)
+exists, err := utils.PathExists(path)
 if err != nil {
 return "", err
 }