allow narrowing of inherited types (#535)

Author: mr-c

Makefile

@@ -34,7 +34,7 @@ COVBASE=coverage run --append
 
 # Updating the Major & Minor version below?
 # Don't forget to update setup.py as well
-VERSION=8.2.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
+VERSION=8.3.$(shell date +%Y%m%d%H%M%S --utc --date=`git log --first-parent \
 	--max-count=1 --format=format:%cI`)
 
 ## all         : default task

schema_salad/avro/schema.py

@@ -303,7 +303,7 @@ def __init__(
                 type_schema = make_avsc_object(atype, names)
             except Exception as e:
                 raise SchemaParseException(
-                    f'Type property "{atype}" not a valid Avro schema.'
+                    f'Type property "{atype}" not a valid Avro schema: {e}'
                 ) from e
         self.set_prop("type", type_schema)
         self.set_prop("name", name)
@@ -409,8 +409,8 @@ def __init__(
                 items_schema = make_avsc_object(items, names)
             except Exception as err:
                 raise SchemaParseException(
-                    f"Items schema ({items}) not a valid Avro schema: (known "
-                    f"names: {list(names.names.keys())})."
+                    f"Items schema ({items}) not a valid Avro schema: {err}. "
+                    f"Known names: {list(names.names.keys())})."
                 ) from err
 
         self.set_prop("items", items_schema)
@@ -451,7 +451,7 @@ def __init__(
                     new_schema = make_avsc_object(schema, names)
                 except Exception as err:
                     raise SchemaParseException(
-                        f"Union item must be a valid Avro schema: {schema}"
+                        f"Union item must be a valid Avro schema: {err}; {schema},"
                     ) from err
             # check the new schema
             if (
@@ -477,7 +477,7 @@ class RecordSchema(NamedSchema):
     def make_field_objects(field_data: List[PropsType], names: Names) -> List[Field]:
         """We're going to need to make message parameters too."""
         field_objects = []  # type: List[Field]
-        field_names = []  # type: List[str]
+        parsed_fields: Dict[str, PropsType] = {}
         for field in field_data:
             if hasattr(field, "get") and callable(field.get):
                 atype = field.get("type")
@@ -504,10 +504,15 @@ def make_field_objects(field_data: List[PropsType], names: Names) -> List[Field]
                     atype, name, has_default, default, order, names, doc, other_props
                 )
                 # make sure field name has not been used yet
-                if new_field.name in field_names:
-                    fail_msg = f"Field name {new_field.name} already in use."
-                    raise SchemaParseException(fail_msg)
-                field_names.append(new_field.name)
+                if new_field.name in parsed_fields:
+                    old_field = parsed_fields[new_field.name]
+                    if not is_subtype(old_field["type"], field["type"]):
+                        raise SchemaParseException(
+                            f"Field name {new_field.name} already in use with "
+                            "incompatible type. "
+                            f"{field['type']} vs {old_field['type']}."
+                        )
+                parsed_fields[new_field.name] = field
             else:
                 raise SchemaParseException(f"Not a valid field: {field}")
             field_objects.append(new_field)
@@ -655,3 +660,62 @@ def make_avsc_object(json_data: JsonDataType, names: Optional[Names] = None) ->
     # not for us!
     fail_msg = f"Could not make an Avro Schema object from {json_data}."
     raise SchemaParseException(fail_msg)
+
+
+def is_subtype(existing: PropType, new: PropType) -> bool:
+    """Checks if a new type specification is compatible with an existing type spec."""
+    if existing == new:
+        return True
+    if isinstance(existing, list) and (new in existing):
+        return True
+    if existing == "Any":
+        if new is None or new == [] or new == ["null"] or new == "null":
+            return False
+        if isinstance(new, list) and "null" in new:
+            return False
+        return True
+    if (
+        isinstance(existing, dict)
+        and "type" in existing
+        and existing["type"] == "array"
+        and isinstance(new, dict)
+        and "type" in new
+        and new["type"] == "array"
+    ):
+        return is_subtype(existing["items"], new["items"])
+    if (
+        isinstance(existing, dict)
+        and "type" in existing
+        and existing["type"] == "enum"
+        and isinstance(new, dict)
+        and "type" in new
+        and new["type"] == "enum"
+    ):
+        return is_subtype(existing["symbols"], new["symbols"])
+    if (
+        isinstance(existing, dict)
+        and "type" in existing
+        and existing["type"] == "record"
+        and isinstance(new, dict)
+        and "type" in new
+        and new["type"] == "record"
+    ):
+        for new_field in cast(List[Dict[str, Any]], new["fields"]):
+            new_field_missing = True
+            for existing_field in cast(List[Dict[str, Any]], existing["fields"]):
+                if new_field["name"] == existing_field["name"]:
+                    if not is_subtype(existing_field["type"], new_field["type"]):
+                        return False
+                    new_field_missing = False
+            if new_field_missing:
+                return False
+        return True
+    if isinstance(existing, list) and isinstance(new, list):
+        missing = False
+        for _type in new:
+            if _type not in existing and (
+                not is_subtype(existing, cast(PropType, _type))
+            ):
+                missing = True
+        return not missing
+    return False

schema_salad/metaschema.py

@@ -22,7 +22,7 @@
     Type,
     Union,
 )
-from urllib.parse import quote, urlsplit, urlunsplit, urlparse
+from urllib.parse import quote, urlparse, urlsplit, urlunsplit
 from urllib.request import pathname2url
 
 from ruamel.yaml.comments import CommentedMap

schema_salad/metaschema/metaschema.yml

@@ -309,7 +309,8 @@ $graph:
       type: boolean?
       doc: |
         If true, this record is abstract and may be used as a base for other
-        records, but is not valid on its own.
+        records, but is not valid on its own. Inherited fields may be
+        re-specified to narrow their type.
 
     - name: extends
       type:
@@ -321,7 +322,7 @@ $graph:
         refScope: 1
       doc: |
         Indicates that this record inherits fields from one or more base records.
-
+        Inherited fields may be re-specified to narrow their type.
     - name: specialize
       type:
         - SpecializeDef[]?

schema_salad/metaschema/salad.md

@@ -9,6 +9,7 @@ Contributors:
 * The developers of Apache Avro
 * The developers of JSON-LD
 * Nebojša Tijanić <nebojsa.tijanic@sbgenomics.com>, Seven Bridges Genomics
+* Michael R. Crusoe, ELIXIR-DE
 
 # Abstract
 
@@ -86,6 +87,13 @@ specification, the following changes have been made:
   is poorly documented, not included in conformance testing,
   and not widely supported.
 
+## Introduction to v1.2
+
+This is the fourth version of the Schema Salad specification. It was created to
+ease the development of extensions to CWL v1.2. The only change is that
+inherited records can narrow the types of fields if those fields are re-specified
+with a matching jsonldPredicate.
+
 ## References to Other Specifications
 
 **Javascript Object Notation (JSON)**: http://json.org

schema_salad/python_codegen_support.py

@@ -19,7 +19,7 @@
     Type,
     Union,
 )
-from urllib.parse import quote, urlsplit, urlunsplit, urlparse
+from urllib.parse import quote, urlparse, urlsplit, urlunsplit
 from urllib.request import pathname2url
 
 from ruamel.yaml.comments import CommentedMap