Merge pull request #1027 from dimitri-yatsenko/master

minor optimization of progress bar in autopopulate

Author: guzman-raphael

.github/workflows/development.yaml

@@ -36,8 +36,6 @@ jobs:
         include:
           - py_ver: "3.7"
             mysql_ver: "5.7"
-          - py_ver: "3.6"
-            mysql_ver: "5.7"
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{matrix.py_ver}}
@@ -106,4 +104,4 @@ jobs:
         with:
           branch: gh-pages
           directory: gh-pages
-          github_token: ${{secrets.GITHUB_TOKEN}}
+          github_token: ${{secrets.GITHUB_TOKEN}}

datajoint/autopopulate.py

@@ -9,6 +9,7 @@
 from .errors import DataJointError, LostConnectionError
 import signal
 import multiprocessing as mp
+import contextlib
 
 # noinspection PyExceptionInherit,PyCallingNonCallable
 
@@ -213,7 +214,7 @@ def handler(signum, frame):
         if not nkeys:
             return
 
-        processes = min(*(_ for _ in (processes, nkeys, mp.cpu_count()) if _))
+        processes = min(_ for _ in (processes, nkeys, mp.cpu_count()) if _)
 
         error_list = []
         populate_kwargs = dict(
@@ -235,17 +236,16 @@ def handler(signum, frame):
             del self.connection._conn.ctx  # SSLContext is not pickleable
             with mp.Pool(
                 processes, _initialize_populate, (self, jobs, populate_kwargs)
-            ) as pool:
-                if display_progress:
-                    with tqdm(desc="Processes: ", total=nkeys) as pbar:
-                        for error in pool.imap(_call_populate1, keys, chunksize=1):
-                            if error is not None:
-                                error_list.append(error)
-                            pbar.update()
-                else:
-                    for error in pool.imap(_call_populate1, keys):
-                        if error is not None:
-                            error_list.append(error)
+            ) as pool, (
+                tqdm(desc="Processes: ", total=nkeys)
+                if display_progress
+                else contextlib.nullcontext()
+            ) as progress_bar:
+                for error in pool.imap(_call_populate1, keys, chunksize=1):
+                    if error is not None:
+                        error_list.append(error)
+                    if display_progress:
+                        progress_bar.update()
             self.connection.connect()  # reconnect parent process to MySQL server
 
         # restore original signal handler:

datajoint/connection.py

@@ -21,6 +21,9 @@
 query_log_max_length = 300
 
 
+cache_key = "query_cache"  # the key to lookup the query_cache folder in dj.config
+
+
 def get_host_hook(host_input):
     if "://" in host_input:
         plugin_name = host_input.split("://")[0]
@@ -220,7 +223,7 @@ def connect(self):
                         k: v
                         for k, v in self.conn_info.items()
                         if k not in ["ssl_input", "host_input"]
-                    }
+                    },
                 )
             except client.err.InternalError:
                 self._conn = client.connect(
@@ -236,7 +239,7 @@ def connect(self):
                             or k == "ssl"
                             and self.conn_info["ssl_input"] is None
                         )
-                    }
+                    },
                 )
         self._conn.autocommit(True)
 
@@ -254,13 +257,12 @@ def set_query_cache(self, query_cache=None):
     def purge_query_cache(self):
         """Purges all query cache."""
         if (
-            "query_cache" in config
-            and isinstance(config["query_cache"], str)
-            and pathlib.Path(config["query_cache"]).is_dir()
+            isinstance(config.get(cache_key), str)
+            and pathlib.Path(config[cache_key]).is_dir()
         ):
-            path_iter = pathlib.Path(config["query_cache"]).glob("**/*")
-            for path in path_iter:
-                path.unlink()
+            for path in pathlib.Path(config[cache_key]).iterdir():
+                if not path.is_dir():
+                    path.unlink()
 
     def close(self):
         self._conn.close()
@@ -313,15 +315,15 @@ def query(
                 "Only SELECT queries are allowed when query caching is on."
             )
         if use_query_cache:
-            if not config["query_cache"]:
+            if not config[cache_key]:
                 raise errors.DataJointError(
-                    "Provide filepath dj.config['query_cache'] when using query caching."
+                    f"Provide filepath dj.config['{cache_key}'] when using query caching."
                 )
             hash_ = uuid_from_buffer(
                 (str(self._query_cache) + re.sub(r"`\$\w+`", "", query)).encode()
                 + pack(args)
             )
-            cache_path = pathlib.Path(config["query_cache"]) / str(hash_)
+            cache_path = pathlib.Path(config[cache_key]) / str(hash_)
             try:
                 buffer = cache_path.read_bytes()
             except FileNotFoundError:

datajoint/table.py

@@ -311,7 +311,7 @@ def update1(self, row):
             raise DataJointError("Update cannot be applied to a restricted table.")
         key = {k: row[k] for k in self.primary_key}
         if len(self & key) != 1:
-            raise DataJointError("Update entry must exist.")
+            raise DataJointError("Update can only be applied to one existing entry.")
         # UPDATE query
         row = [
             self.__make_placeholder(k, v)

setup.py

@@ -3,7 +3,7 @@
 from os import path
 import sys
 
-min_py_version = (3, 6)
+min_py_version = (3, 7)
 
 if sys.version_info <  min_py_version:
     sys.exit('DataJoint is only supported for Python {}.{} or higher'.format(*min_py_version))

tests/schema_simple.py

@@ -200,9 +200,8 @@ class Website(dj.Part):
         """
 
     def populate_random(self, n=10):
-        faker.Faker.seed(0)
         fake = faker.Faker()
-        faker.Faker.seed(0)  # make tests deterministic
+        faker.Faker.seed(0)  # make test deterministic
         for _ in range(n):
             profile = fake.profile()
             with self.connection.transaction: