Merge pull request #506 from dimitri-yatsenko/master

Implement projected dependencies. Fix #436

Author: eywalker

datajoint/declare.py

@@ -16,7 +16,9 @@
 logger = logging.getLogger(__name__)
 
 
-def build_foreign_key_parser():
+def build_foreign_key_parser_old():
+    # old-style foreign key parser. Superceded by expression-based syntax. See issue #436
+    # This will be deprecated in a future release.
     left = pp.Literal('(').suppress()
     right = pp.Literal(')').suppress()
     attribute_name = pp.Word(pp.srange('[a-z]'), pp.srange('[a-z0-9_]'))
@@ -31,6 +33,16 @@ def build_foreign_key_parser():
     return new_attrs + arrow + options + ref_table + ref_attrs
 
 
+def build_foreign_key_parser():
+    arrow = pp.Literal('->').suppress()
+    lbracket = pp.Literal('[').suppress()
+    rbracket = pp.Literal(']').suppress()
+    option = pp.Word(pp.srange('[a-zA-Z]'))
+    options = pp.Optional(lbracket + pp.delimitedList(option) + rbracket).setResultsName('options')
+    ref_table = pp.restOfLine.setResultsName('ref_table')
+    return arrow + options + ref_table
+
+
 def build_attribute_parser():
     quoted = pp.Or(pp.QuotedString('"'), pp.QuotedString("'"))
     colon = pp.Literal(':').suppress()
@@ -50,6 +62,7 @@ def build_index_parser():
     return unique + index + left + pp.delimitedList(attribute_name).setResultsName('attr_list') + right
 
 
+foreign_key_parser_old = build_foreign_key_parser_old()
 foreign_key_parser = build_foreign_key_parser()
 attribute_parser = build_attribute_parser()
 index_parser = build_index_parser()
@@ -77,16 +90,22 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
     """
     # Parse and validate
     from .table import Table
+    from .query import Projection
+
+    new_style = True   # See issue #436.  Old style to be deprecated in a future release
     try:
         result = foreign_key_parser.parseString(line)
-    except pp.ParseException as err:
-        raise DataJointError('Parsing error in line "%s". %s.' % (line, err))
+    except pp.ParseException:
+        try:
+            result = foreign_key_parser_old.parseString(line)
+        except pp.ParseBaseException as err:
+            raise DataJointError('Parsing error in line "%s". %s.' % (line, err)) from None
+        else:
+            new_style = False
     try:
-        referenced_class = eval(result.ref_table, context)
-    except NameError:
+        ref = eval(result.ref_table, context)
+    except Exception if new_style else NameError:
         raise DataJointError('Foreign key reference %s could not be resolved' % result.ref_table)
-    if not issubclass(referenced_class, Table):
-        raise DataJointError('Foreign key reference %s must be a subclass of UserTable' % result.ref_table)
 
     options = [opt.upper() for opt in result.options]
     for opt in options:  # check for invalid options
@@ -97,65 +116,75 @@ def compile_foreign_key(line, context, attributes, primary_key, attr_sql, foreig
     if is_nullable and primary_key is not None:
         raise DataJointError('Primary dependencies cannot be nullable in line "{line}"'.format(line=line))
 
-    ref = referenced_class()
-    if not all(r in ref.primary_key for r in result.ref_attrs):
-        raise DataJointError('Invalid foreign key attributes in "%s"' % line)
-
-    try:
-        raise DataJointError('Duplicate attributes "{attr}" in "{line}"'.format(
-            attr=next(attr for attr in result.new_attrs if attr in attributes),
-            line=line))
-    except StopIteration:
-        pass   # the normal outcome
-
-    # Match the primary attributes of the referenced table to local attributes
-    new_attrs = list(result.new_attrs)
-    ref_attrs = list(result.ref_attrs)
-
-    # special case, the renamed attribute is implicit
-    if new_attrs and not ref_attrs:
-        if len(new_attrs) != 1:
-            raise DataJointError('Renamed foreign key must be mapped to the primary key in "%s"' % line)
-        if len(ref.primary_key) == 1:
-            # if the primary key has one attribute, allow implicit renaming
-            ref_attrs = ref.primary_key
-        else:
-            # if only one primary key attribute remains, then allow implicit renaming
-            ref_attrs = [attr for attr in ref.primary_key if attr not in attributes]
-            if len(ref_attrs) != 1:
-                raise DataJointError('Could not resovle which primary key attribute should be referenced in "%s"' % line)
-
-    if len(new_attrs) != len(ref_attrs):
-        raise DataJointError('Mismatched attributes in foreign key "%s"' % line)
-
-    # expand the primary key of the referenced table
-    lookup = dict(zip(ref_attrs, new_attrs)).get  # from foreign to local
-    ref_attrs = [attr for attr in ref.primary_key if lookup(attr, attr) not in attributes]
-    new_attrs = [lookup(attr, attr) for attr in ref_attrs]
-
-    # sanity check
-    assert len(new_attrs) == len(ref_attrs) and not any(attr in attributes for attr in new_attrs)
+    if not new_style:
+        if not isinstance(ref, type) or not issubclass(ref, Table):
+            raise DataJointError('Foreign key reference %r must be a valid query' % result.ref_table)
+
+    if isinstance(ref, type) and issubclass(ref, Table):
+        ref = ref()
+
+    # check that dependency is of supported type
+    if (not isinstance(ref, (Table, Projection)) or len(ref.restriction) or
+            (isinstance(ref, Projection) and (not isinstance(ref._arg, Table) or len(ref._arg.restriction)))):
+        raise DataJointError('Dependency "%s" is not supported (yet). Use a base table or its projection.' %
+                             result.ref_table)
+
+    if not new_style:
+        # for backward compatibility with old-style dependency declarations.  See issue #436
+        if not isinstance(ref, Table):
+            DataJointError('Dependency "%s" is not supported. Check documentation.' % result.ref_table)
+        if not all(r in ref.primary_key for r in result.ref_attrs):
+            raise DataJointError('Invalid foreign key attributes in "%s"' % line)
+        try:
+            raise DataJointError('Duplicate attributes "{attr}" in "{line}"'.format(
+                attr=next(attr for attr in result.new_attrs if attr in attributes),
+                line=line))
+        except StopIteration:
+            pass   # the normal outcome
+
+        # Match the primary attributes of the referenced table to local attributes
+        new_attrs = list(result.new_attrs)
+        ref_attrs = list(result.ref_attrs)
+
+        # special case, the renamed attribute is implicit
+        if new_attrs and not ref_attrs:
+            if len(new_attrs) != 1:
+                raise DataJointError('Renamed foreign key must be mapped to the primary key in "%s"' % line)
+            if len(ref.primary_key) == 1:
+                # if the primary key has one attribute, allow implicit renaming
+                ref_attrs = ref.primary_key
+            else:
+                # if only one primary key attribute remains, then allow implicit renaming
+                ref_attrs = [attr for attr in ref.primary_key if attr not in attributes]
+                if len(ref_attrs) != 1:
+                    raise DataJointError('Could not resovle which primary key attribute should be referenced in "%s"' % line)
+
+        if len(new_attrs) != len(ref_attrs):
+            raise DataJointError('Mismatched attributes in foreign key "%s"' % line)
+
+        if ref_attrs:
+            ref = ref.proj(**dict(zip(new_attrs, ref_attrs)))
 
     # declare new foreign key attributes
-    for ref_attr in ref_attrs:
-        new_attr = lookup(ref_attr, ref_attr)
-        attributes.append(new_attr)
-        if primary_key is not None:
-            primary_key.append(new_attr)
-        attr_sql.append(
-            ref.heading[ref_attr].sql.replace(ref_attr, new_attr, 1).replace('NOT NULL', '', int(is_nullable)))
+    base = ref._arg if isinstance(ref, Projection) else ref   # base reference table
+    for attr, ref_attr in zip(ref.primary_key, base.primary_key):
+        if attr not in attributes:
+            attributes.append(attr)
+            if primary_key is not None:
+                primary_key.append(attr)
+            attr_sql.append(
+                base.heading[ref_attr].sql.replace(ref_attr, attr, 1).replace('NOT NULL ', '', int(is_nullable)))
 
     # declare the foreign key
     foreign_key_sql.append(
         'FOREIGN KEY (`{fk}`) REFERENCES {ref} (`{pk}`) ON UPDATE CASCADE ON DELETE RESTRICT'.format(
-            fk='`,`'.join(lookup(attr, attr) for attr in ref.primary_key),
-            pk='`,`'.join(ref.primary_key),
-            ref=ref.full_table_name))
+            fk='`,`'.join(ref.primary_key),
+            pk='`,`'.join(base.primary_key),
+            ref=base.full_table_name))
 
     # declare unique index
     if is_unique:
-        index_sql.append('UNIQUE INDEX ({attrs})'.format(
-            attrs='`,`'.join(lookup(attr, attr) for attr in ref.primary_key)))
+        index_sql.append('UNIQUE INDEX ({attrs})'.format(attrs='`,`'.join(ref.primary_key)))
 
 
 def declare(full_table_name, definition, context):
@@ -223,7 +252,6 @@ def compile_attribute(line, in_key, foreign_key_sql):
     :param foreign_key_sql:
     :returns: (name, sql, is_external) -- attribute name and sql code for its declaration
     """
-
     try:
         match = attribute_parser.parseString(line+'#', parseAll=True)
     except pp.ParseException as err:

datajoint/query.py

@@ -52,6 +52,10 @@ def append(self, restriction):
             super().append(restriction)
 
 
+def is_true(restriction):
+    return restriction is True or isinstance(restriction, AndList) and not len(restriction)
+
+
 class Query:
     """
     Query implements the relational algebra.
@@ -249,6 +253,8 @@ def __iand__(self, restriction):
 
         See query.restrict for more detail.
         """
+        if is_true(restriction):
+            return self
         return (Subquery.create(self) if self.heading.expressions else self).restrict(restriction)
 
     def __and__(self, restriction):
@@ -258,7 +264,7 @@ def __and__(self, restriction):
         See query.restrict for more detail.
         """
         return (Subquery.create(self)  # the HAVING clause in GroupBy can handle renamed attributes but WHERE cannot
-                if self.heading.expressions and not isinstance(self, GroupBy)
+                if not(is_true(restriction)) and self.heading.expressions and not isinstance(self, GroupBy)
                 else self.__class__(self)).restrict(restriction)
 
     def __isub__(self, restriction):
@@ -324,8 +330,8 @@ def restrict(self, restriction):
         :param restriction: a sequence or an array (treated as OR list), another relation, an SQL condition string, or
             an AndList.
         """
-        assert not self.heading.expressions or isinstance(self, GroupBy), "Cannot restrict a projection" \
-                                                                          " with renamed attributes in place."
+        assert is_true(restriction) or not self.heading.expressions or isinstance(self, GroupBy), \
+            "Cannot restrict a projection with renamed attributes in place."
         self.restriction.append(restriction)
         return self
 

datajoint/schema.py

@@ -5,7 +5,6 @@
 import re
 import itertools
 import collections
-import types
 from . import conn, config
 from .errors import DataJointError
 from .jobs import JobTable

datajoint/table.py

@@ -488,15 +488,14 @@ def describe(self, context=None, printout=True):
                                 props=index_props,
                                 class_name=lookup_class_name(parent_name, context) or parent_name)
                         else:
-                            # expression foreign key
+                            # projected foreign key
                             parent_name = list(self.connection.dependencies.in_edges(parent_name))[0][0]
                             lst = [(attr, ref) for attr, ref in fk_props['attr_map'].items() if ref != attr]
-                            definition += '({attr_list}) ->{props} {class_name}{ref_list}\n'.format(
+                            definition += '->{props} {class_name}.proj({proj_list})\n'.format(
                                 attr_list=', '.join(r[0] for r in lst),
                                 props=index_props,
                                 class_name=lookup_class_name(parent_name, context) or parent_name,
-                                ref_list=('' if len(attributes_thus_far) - len(attributes_declared) == 1
-                                          else '(%s)' % ','.join(r[1] for r in lst)))
+                                proj_list=','.join('{}="{}"'.format(a,b) for a, b in lst))
                             attributes_declared.update(fk_props['attr_map'])
             if do_include:
                 attributes_declared.add(attr.name)

tests/schema.py

@@ -260,7 +260,7 @@ class SigTermTable(dj.Computed):
     -> SimpleSource
     """
 
-    def _make_tuples(self, key):
+    def make(self, key):
         os.kill(os.getpid(), signal.SIGTERM)
 
 
@@ -271,12 +271,13 @@ class DecimalPrimaryKey(dj.Lookup):
     """
     contents = zip((0.1, 0.25, 3.99))
 
+
 @schema
 class IndexRich(dj.Manual):
     definition = """
     -> Experiment
     ---
-    (first) ->[unique, nullable] User
+    -> [unique, nullable] User.proj(first="username")
     first_date : date
     value : int
     index (first_date, value)

tests/schema_advanced.py

@@ -94,7 +94,7 @@ class Cell(dj.Manual):
 class InputCell(dj.Manual):
     definition = """  # a synapse within the slice
     -> Cell
-    (input)-> Cell(cell)
+    -> Cell.proj(input="cell")
     """
 
 
@@ -108,8 +108,9 @@ class LocalSynapse(dj.Manual):
 
 @schema
 class GlobalSynapse(dj.Manual):
+    # Mix old-style and new-style projected foreign keys
     definition = """
     # a synapse within the slice
-    (pre_slice, pre_cell) -> Cell(slice, cell)
+    -> Cell.proj(pre_slice="slice", pre_cell="cell")
     (post_slice, post_cell)-> Cell(slice, cell)
     """