Merge pull request #9 from supercontainers/update-new-graph-specs

update create command for new spec

Author: vsoch

cmd/compspec/create/create.go

@@ -52,6 +52,10 @@ func Run(specname string, fields []string, saveto string) error {
 
 	// The compspec returned is the populated Compatibility request!
 	compspec, err := PopulateExtractors(&result, request)
+	if err != nil {
+		return err
+	}
+
 	output, err := compspec.ToJson()
 	if err != nil {
 		return err
@@ -71,43 +75,58 @@ func Run(specname string, fields []string, saveto string) error {
 // After this we can save the populated thing into an artifact (json DUMP)
 func PopulateExtractors(result *p.Result, request *types.CompatibilityRequest) (*types.CompatibilityRequest, error) {
 
+	// Every metadata attribute must be known under a schema
+	schemas := request.Metadata.Schemas
+	if len(schemas) == 0 {
+		return nil, fmt.Errorf("the request must have one or more schemas")
+	}
 	for i, compat := range request.Compatibilities {
-		for key, extractorKey := range compat.Annotations {
+
+		// The compatibility section name is a schema, and must be defined
+		url, ok := schemas[compat.Name]
+		if !ok {
+			return nil, fmt.Errorf("%s is missing a schema", compat.Name)
+		}
+		if url == "" {
+			return nil, fmt.Errorf("%s has an empty schema", compat.Name)
+		}
+
+		for key, extractorKey := range compat.Attributes {
 
 			// Get the extractor, section, and subfield from the extractor lookup key
 			f, err := p.ParseField(extractorKey)
 			if err != nil {
 				fmt.Printf("warning: cannot parse %s: %s, setting to empty\n", key, extractorKey)
-				compat.Annotations[key] = ""
+				compat.Attributes[key] = ""
 				continue
 			}
 
 			// If we get here, we can parse it and look it up in our result metadata
 			extractor, ok := result.Results[f.Extractor]
 			if !ok {
 				fmt.Printf("warning: extractor %s is unknown, setting to empty\n", f.Extractor)
-				compat.Annotations[key] = ""
+				compat.Attributes[key] = ""
 				continue
 			}
 
 			// Now get the section
 			section, ok := extractor.Sections[f.Section]
 			if !ok {
 				fmt.Printf("warning: section %s.%s is unknown, setting to empty\n", f.Extractor, f.Section)
-				compat.Annotations[key] = ""
+				compat.Attributes[key] = ""
 				continue
 			}
 
 			// Now get the value!
 			value, ok := section[f.Field]
 			if !ok {
 				fmt.Printf("warning: field %s.%s.%s is unknown, setting to empty\n", f.Extractor, f.Section, f.Field)
-				compat.Annotations[key] = ""
+				compat.Attributes[key] = ""
 				continue
 			}
 
 			// If we get here - we found it! Hooray!
-			compat.Annotations[key] = value
+			compat.Attributes[key] = value
 		}
 
 		// Update the compatibiity

docs/usage.md

@@ -91,31 +91,32 @@ The idea here is that you can add custom metadata fields during your build, whic
   "kind": "CompatibilitySpec",
   "metadata": {
     "name": "lammps-prototype",
-    "jsonSchema": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+    "schemas": {
+      "archspec.io": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
+      "org.supercontainers": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+    }
   },
   "compatibilities": [
     {
-      "name": "org.supercontainers.mpi",
+      "name": "org.supercontainers",
       "version": "0.0.0",
-      "annotations": {
-        "implementation": "mpich",
-        "version": "4.1.1"
+      "attributes": {
+        "hardware.gpu.available": "yes",
+        "mpi.implementation": "mpich",
+        "mpi.version": "4.1.1",
+        "os.name": "Ubuntu 22.04.3 LTS",
+        "os.release": "22.04.3",
+        "os.vendor": "ubuntu",
+        "os.version": "22.04"
       }
     },
     {
-      "name": "org.supercontainers.hardware.gpu",
+      "name": "archspec.io",
       "version": "0.0.0",
-      "annotations": {
-        "available": "yes"
-      }
-    },
-    {
-      "name": "io.archspec.cpu",
-      "version": "0.0.0",
-      "annotations": {
-        "model": "13th Gen Intel(R) Core(TM) i5-1335U",
-        "target": "amd64",
-        "vendor": "GenuineIntel"
+      "attributes": {
+        "cpu.model": "13th Gen Intel(R) Core(TM) i5-1335U",
+        "cpu.target": "amd64",
+        "cpu.vendor": "GenuineIntel"
       }
     }
   ]
@@ -136,7 +137,7 @@ For now we will manually remember the pairing, at least until the compatibility
 
 Check is the command you would use to check a potential host against one or more existing artifacts.
 For a small experiment of using create against a set of containers and then testing how to do a check, we are going to place content
-in [examples/check-lammps](examples/check-lammps). As an example, we might use the manifest in that directory to run a check.
+in [examples/check-lammps](examples/check-lammps). Note that we generated the actual compatibility spec and pushed with oras before running the example here! Following that, we might use the manifest in that directory to run a check.
 Note that since most use cases aren't checking the images in the manifest list against the host running the command, we instead
 provide the parameters about the expected runtime host to them.
 

examples/check-lammps/generate-artifact.sh

@@ -9,7 +9,7 @@ wget --quiet https://github.com/supercontainers/compspec-go/releases/download/1-
 chmod +x compspec
 
 # Download the spec for our compatibility artifact
-wget --quiet https://gist.githubusercontent.com/vsoch/fcd0f7d633860674cb085a8540ce4bb2/raw/880f3764b9394ccaa21fd768b235c7a89609aa65/lammps-experiment.yaml
+wget --quiet https://gist.githubusercontent.com/vsoch/fcd0f7d633860674cb085a8540ce4bb2/raw/4f8e730f1d74c070e63de79bf8b6f86a528ef1c9/lammps-experiment.yaml
 
 # Generate!
 ./compspec create --in ./lammps-experiment.yaml -a custom.gpu.available=$hasGpu -o ${path}

examples/generated-compatibility-spec.json

@@ -3,41 +3,32 @@
   "kind": "CompatibilitySpec",
   "metadata": {
     "name": "lammps-prototype",
-    "jsonSchema": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+    "schemas": {
+      "archspec.io": "https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json",
+      "org.supercontainers": "https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json"
+    }
   },
   "compatibilities": [
     {
-      "name": "org.supercontainers.mpi",
-      "version": "0.0.0",
-      "annotations": {
-        "implementation": "mpich",
-        "version": "4.1.1"
-      }
-    },
-    {
-      "name": "org.supercontainers.os",
-      "version": "0.0.0",
-      "annotations": {
-        "name": "Ubuntu 22.04.3 LTS",
-        "release": "22.04.3",
-        "vendor": "ubuntu",
-        "version": "22.04"
-      }
-    },
-    {
-      "name": "org.supercontainers.hardware.gpu",
+      "name": "org.supercontainers",
       "version": "0.0.0",
-      "annotations": {
-        "available": "yes"
+      "attributes": {
+        "hardware.gpu.available": "yes",
+        "mpi.implementation": "mpich",
+        "mpi.version": "4.1.1",
+        "os.name": "Ubuntu 22.04.3 LTS",
+        "os.release": "22.04.3",
+        "os.vendor": "ubuntu",
+        "os.version": "22.04"
       }
     },
     {
-      "name": "io.archspec.cpu",
+      "name": "archspec.io",
       "version": "0.0.0",
-      "annotations": {
-        "model": "13th Gen Intel(R) Core(TM) i5-1335U",
-        "target": "amd64",
-        "vendor": "GenuineIntel"
+      "attributes": {
+        "cpu.model": "13th Gen Intel(R) Core(TM) i5-1335U",
+        "cpu.target": "amd64",
+        "cpu.vendor": "GenuineIntel"
       }
     }
   ]

examples/lammps-experiment.yaml

@@ -7,38 +7,30 @@ kind: CompatibilitySpec
 metadata:
   name: lammps-prototype
 
-  # "Validate the final compatibility spec against this schema'
-  jsonSchema: https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json
+  # "Validate the namespaced attributes with these schemas"
+  schemas:
+    org.supercontainers: https://raw.githubusercontent.com/supercontainers/compspec/main/supercontainers/compspec.json
+    archspec.io: https://raw.githubusercontent.com/supercontainers/compspec/main/archspec/compspec.json
 
 # These are not values, but mappings, from an extractor into the compspec we want
 compatibilities:
-- name: "org.supercontainers.mpi" 
+- name: "org.supercontainers" 
   version: "0.0.0"
-  annotations:
-    implementation: library.mpi.variant
-    version: library.mpi.version
-
-- name: "org.supercontainers.os" 
-  version: "0.0.0"
-  annotations:
-    name: system.os.name
-    release: system.os.release
-    vendor: system.os.vendor
-    version: system.os.version
-
-# This is an example of a custom metadata attribute provided by the user (commandline)
-# It can override an actual attribute, or just be random / new, -o custom.gpu.available=yes
-- name: "org.supercontainers.hardware.gpu"
-  version: "0.0.0"
-  annotations:
-    available: custom.gpu.available
+  attributes:
+    mpi.implementation: library.mpi.variant
+    mpi.version: library.mpi.version
+    os.name: system.os.name
+    os.release: system.os.release
+    os.vendor: system.os.vendor
+    os.version: system.os.version
+    hardware.gpu.available: custom.gpu.available
 
 # Note that for now we are using the processor in index 0 to represent all
 # I'm not sure about cases where this set isn't homogeneous!
 # Since target is part of the container build, we will provide it
-- name: "io.archspec.cpu"
+- name: "archspec.io"
   version: "0.0.0"
-  annotations:
-    model: system.processor.0.model
-    target: system.arch.name
-    vendor: system.processor.0.vendor
+  attributes:
+    cpu.model: system.processor.0.model
+    cpu.target: system.arch.name
+    cpu.vendor: system.processor.0.vendor

pkg/types/types.go

@@ -10,11 +10,11 @@ type CompatibilitySpec struct {
 	Compatibilities map[string]CompatibilitySpec `json:"compatibilities"`
 }
 type CompatibiitySpec struct {
-	Version     string      `json:"version"`
-	Annotations Annotations `json:"annotations"`
+	Version    string     `json:"version"`
+	Attributes Attributes `json:"attributes"`
 }
 
-type Annotations map[string]string
+type Attributes map[string]string
 
 // A compatibility request is a mapping between a user preferences (some request to create a
 // compatibility artifact) to a set of metadata attributes known by extractors.
@@ -27,16 +27,16 @@ type CompatibilityRequest struct {
 	Compatibilities []CompatibilityMapping `json:"compatibilities,omitempty"`
 }
 type Metadata struct {
-	Name       string `json:"name,omitempty"`
-	JSONSchema string `json:"jsonSchema,omitempty"`
+	Name    string            `json:"name,omitempty"`
+	Schemas map[string]string `json:"schemas,omitempty"`
 }
 
 // A compatibility mapping has one or more annotations that convert
 // between extractor and compspec.json (the JsonSchema provided above)
 type CompatibilityMapping struct {
-	Name        string            `json:"name,omitempty"`
-	Version     string            `json:"version,omitempty"`
-	Annotations map[string]string `json:"annotations,omitempty"`
+	Name       string            `json:"name,omitempty"`
+	Version    string            `json:"version,omitempty"`
+	Attributes map[string]string `json:"attributes,omitempty"`
 }
 
 // ToJson dumps our request to json for the artifact
@@ -53,7 +53,7 @@ func (r *CompatibilityRequest) GetExtractors() []string {
 
 	set := map[string]bool{}
 	for _, compat := range r.Compatibilities {
-		for _, request := range compat.Annotations {
+		for _, request := range compat.Attributes {
 
 			// The extractor name is the first field
 			parts := strings.Split(request, ".")