Merge branch 'master' of https://github.com/datajoint/datajoint-python into revamp-delete

Author: dimitri-yatsenko

datajoint/autopopulate.py

@@ -69,6 +69,9 @@ def _jobs_to_do(self, restrictions):
         """
         :return: the relation containing the keys to be computed (derived from self.key_source)
         """
+        if self.restriction:
+            raise DataJointError('Cannot call populate on a restricted table. '
+                                 'Instead, pass conditions to populate() as arguments.')
         todo = self.key_source
         if not isinstance(todo, RelationalOperand):
             raise DataJointError('Invalid key_source value')

datajoint/base_relation.py

@@ -463,7 +463,7 @@ def describe(self, printout=True):
                                 class_name=lookup_class_name(parent_name, self.context) or parent_name)
                         else:
                             # aliased foreign key
-                            parent_name = self.connection.dependencies.in_edges(parent_name)[0][0]
+                            parent_name = list(self.connection.dependencies.in_edges(parent_name))[0][0]
                             lst = [(attr, ref) for attr, ref in fk_props['attr_map'].items() if ref != attr]
                             definition += '({attr_list}) -> {class_name}{ref_list}\n'.format(
                                 attr_list=','.join(r[0] for r in lst),

datajoint/connection.py

@@ -122,10 +122,8 @@ def query(self, query, args=(), as_dict=False, suppress_warnings=True):
         cursor = client.cursors.DictCursor if as_dict else client.cursors.Cursor
         cur = self._conn.cursor(cursor=cursor)
 
+        logger.debug("Executing SQL:" + query[0:300])
         try:
-            # Log the query
-            logger.debug("Executing SQL:" + query[0:300])
-
             with warnings.catch_warnings():
                 if suppress_warnings:
                     # suppress all warnings arising from underlying SQL library
@@ -144,9 +142,11 @@ def query(self, query, args=(), as_dict=False, suppress_warnings=True):
             else:
                 raise
         except err.ProgrammingError as e:
-            print('Error in query:')
-            print(query)
-            raise
+            raise DataJointError("\n".join((
+                "Error in query:", query,
+                "Please check spelling, syntax, and existence of tables and attributes.",
+                "When restricting a relation by a condition in a string, enclose attributes in backquotes."
+            )))
         return cur
 
     def get_user(self):

datajoint/dependencies.py

@@ -7,8 +7,13 @@
 class Dependencies(nx.DiGraph):
     """
     The graph of dependencies (foreign keys) between loaded tables.
+
+    Note: the 'connnection' argument should normally be supplied;
+    Empty use is permitted to facilliate use of networkx algorithms which
+    internally create objects with the expectation of empty constructors.
+    See also: https://github.com/datajoint/datajoint-python/pull/443
     """
-    def __init__(self, connection):
+    def __init__(self, connection=None):
         self._conn = connection
         self._node_alias_count = itertools.count()
         super().__init__(self)
@@ -100,5 +105,9 @@ def descendants(self, full_table_name):
         :param full_table_name:  In form `schema`.`table_name`
         :return: all dependent tables sorted in topological order.  Self is included.
         """
-        nodes = nx.algorithms.dag.descendants(self, full_table_name)
-        return [full_table_name] + nx.algorithms.dag.topological_sort(self, nodes)
+
+        nodes = self.subgraph(
+            nx.algorithms.dag.descendants(self, full_table_name))
+
+        return [full_table_name] + list(
+            nx.algorithms.dag.topological_sort(nodes))

datajoint/erd.py

@@ -206,8 +206,8 @@ def _make_graph(self):
                 nx.algorithms.boundary.node_boundary(nx.DiGraph(self).reverse(), self.nodes_to_show))
             nodes = self.nodes_to_show.union(a for a in gaps if a.isdigit)
             # construct subgraph and rename nodes to class names
-            graph = nx.DiGraph(self).subgraph(nodes)
-            nx.set_node_attributes(graph, 'node_type', {n: _get_tier(n) for n in graph})
+            graph = nx.DiGraph(nx.DiGraph(self).subgraph(nodes))
+            nx.set_node_attributes(graph, name='node_type', values={n: _get_tier(n) for n in graph})
             # relabel nodes to class names
             clean_context = dict((k, v) for k, v in self.context.items()
                                  if not k.startswith('_'))  # exclude ipython's implicit variables

datajoint/external.py

@@ -77,6 +77,8 @@ def get(self, blob_hash):
         get an object from external store.
         Does not need to check whether it's in the table.
         """
+        if blob_hash is None:
+            return None
         store = blob_hash[STORE_HASH_LENGTH:]
         store = 'external' + ('-' if store else '') + store
 
@@ -184,4 +186,3 @@ def _get_store_spec(store):
             raise DataJointError(
                 'Unknown external storage protocol "{protocol}" in "{store}"'.format(store=store, **spec))
         return spec
-

datajoint/jobs.py

@@ -1,3 +1,4 @@
+from _decimal import Decimal
 from .hash import key_hash
 import os
 import pymysql
@@ -58,6 +59,13 @@ def drop(self):
         """bypass interactive prompts and dependencies"""
         self.drop_quick()
 
+    @staticmethod
+    def packable_or_none(key):
+        for v in key.values():
+            if isinstance(v, Decimal):
+                return None
+        return key
+
     def reserve(self, table_name, key):
         """
         Reserve a job for computation.  When a job is reserved, the job table contains an entry for the
@@ -73,7 +81,7 @@ def reserve(self, table_name, key):
             host=os.uname().nodename,
             pid=os.getpid(),
             connection_id=self.connection.connection_id,
-            key=key,
+            key=self.packable_or_none(key),
             user=self._user)
         try:
             self.insert1(job, ignore_extra_fields=True)
@@ -109,7 +117,7 @@ def error(self, table_name, key, error_message, error_stack=None):
                  pid=os.getpid(),
                  connection_id=self.connection.connection_id,
                  user=self._user,
-                 key=key,
+                 key=self.packable_or_none(key),
                  error_message=error_message,
                  error_stack=error_stack),
             replace=True, ignore_extra_fields=True)

datajoint/s3.py

@@ -16,9 +16,6 @@ def __init__(self, endpoint, bucket, access_key, secret_key, location, database,
         self.bucket = bucket
         self.remote_path = '/'.join((location.lstrip('/'), database))
 
-    def make_bucket(self):
-        self.client.make_bucket(self.bucket)
-
     def put(self, blob_hash, blob):
         try:
             self.client.put_object(self.bucket, '/'.join((self.remote_path, blob_hash)), BytesIO(blob), len(blob))

datajoint/user_relations.py

@@ -12,7 +12,7 @@
 
 # attributes that trigger instantiation of user classes
 supported_class_attrs = set((
-    'key_source', 'describe', 'populate', 'progress',
+    'key_source', 'describe', 'populate', 'progress', 'primary_key',
     'proj', 'aggr', 'heading', 'fetch', 'fetch1',
     'insert', 'insert1', 'drop', 'drop_quick',
     'delete', 'delete_quick'))

requirements.txt

@@ -3,6 +3,6 @@ pymysql>=0.7.2
 pyparsing
 ipython
 tqdm
-networkx~=1.11
-pydotplus
+networkx
+pydot
 minio