config_checker update and linting for C408,C419,C405, C409, E721,F401,E722

.flake8

@@ -0,0 +1,23 @@
+[flake8]
+extend-ignore =
+    E114,
+    E115,
+    E116,
+    E201,
+    E202,
+    E203,
+    E204,
+    E231,
+    E265,
+    E266,
+    E303,
+    E402,
+    E501,
+exclude =
+    ; __init__.py # totally ignore __init__.py files
+    setup.py # ignore setup.py file
+    docs/
+#F401 ignore unused imports in __init__.py files
+#F403 ignore unable to detect undefined names from import *
+per-file-ignores =
+    __init__.py:F401,F403

.github/workflows/build-flake.yml

@@ -1,40 +0,0 @@
-# This workflow will install Python dependencies, run tests and lint with a single version of Python
-# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
-
-name: Check module can be imported
-
-on:
-  push:
-    branches: [ "dev" ]
-  pull_request:
-    branches: [ "dev" ]
-
-permissions:
-  contents: read
-
-jobs:
-  build:
-
-    runs-on: ubuntu-latest
-
-    steps:
-    - uses: actions/checkout@v4
-    - name: Set up Python 3.10
-      uses: actions/setup-python@v5
-      with:
-        python-version: "3.10"
-    - name: Install dependencies
-      run: |
-        python -m pip install --upgrade pip
-        pip install flake8 pytest
-        if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
-    - name: Lint with flake8
-      run: |
-        # stop the build if there are Python syntax errors or undefined names
-        flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
-        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
-        # flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
-    - name: Test that the module imports
-      run: |
-        pip install .
-        python -c "import py4DSTEM; print(py4DSTEM.__version__)"

.github/workflows/linter.yml

@@ -1,4 +1,4 @@
-name: Check for errors with flake8
+name: Lint with super-linter@v5-slim
 
 on:
   push:
@@ -17,9 +17,14 @@ jobs:
           fetch-depth: 0
 
       - name: Lint Code Base
-        uses: github/super-linter@v5
+        uses: super-linter/super-linter/slim@v5 # updated to latest slim as quicker to download 
         env:
-          VALIDATE_ALL_CODEBASE: false
-          VALIDATE_PYTHON_FLAKE8: true
-          DEFAULT_BRANCH: "dev"
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          VALIDATE_ALL_CODEBASE: false # only check changes
+          VALIDATE_PYTHON_FLAKE8: true # lint with flake8
+          DEFAULT_BRANCH: "dev" # set default branch to dev
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for github things
+          # FILTER_REGEX_EXCLUDE: .*test/.* # exclude test dirs
+          FILTER_REGEX_EXCLUDE: .*__init__.py/.* # exclude test dirs
+          FILTER_REGEX_INCLUDE: .*py4DSTEM/.* # only look for py4DSTEM
+          LINTER_RULES_PATH: / # set toplevel dir as the path to look for rules
+          PYTHON_FLAKE8_CONFIG_FILE: .flake8 # set specific config file

py4DSTEM/braggvectors/braggvectors.py

@@ -1,7 +1,7 @@
 # Defines the BraggVectors class
 
 from py4DSTEM.data import Data
-from emdfile import Custom, PointListArray, PointList, Metadata
+from emdfile import Custom, PointListArray, Metadata
 from py4DSTEM.braggvectors.braggvector_methods import BraggVectorMethods
 from os.path import basename
 import numpy as np

py4DSTEM/braggvectors/diskdetection.py

@@ -221,7 +221,7 @@ def find_Bragg_disks(
                 mode = "dp"
             elif data.ndim == 3:
                 mode = "dp_stack"
-        except:
+        except Exception:
             er = f"entry {data} for `data` could not be parsed"
             raise Exception(er)
 

py4DSTEM/braggvectors/diskdetection_aiml.py

@@ -4,18 +4,15 @@
 """
 
 import os
-import glob
 import json
 import shutil
 import numpy as np
 from pathlib import Path
 
 
-from scipy.ndimage import gaussian_filter
 from time import time
-from numbers import Number
 
-from emdfile import tqdmnd, PointList, PointListArray
+from emdfile import tqdmnd, PointListArray
 from py4DSTEM.braggvectors.braggvectors import BraggVectors
 from py4DSTEM.data import QPoints
 from py4DSTEM.process.utils import get_maxima_2D
@@ -105,12 +102,14 @@ def find_Bragg_disks_aiml_single_DP(
     """
     try:
         import crystal4D
-    except:
-        raise ImportError("Import Error: Please install crystal4D before proceeding")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "Import Error: Please install crystal4D before proceeding"
+        )
     try:
         import tensorflow as tf
-    except:
-        raise ImportError(
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
             "Please install tensorflow before proceeding - please check "
             + "https://www.tensorflow.org/install"
             + "for more information"
@@ -258,8 +257,10 @@ def find_Bragg_disks_aiml_selected(
 
     try:
         import crystal4D
-    except:
-        raise ImportError("Import Error: Please install crystal4D before proceeding")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "Import Error: Please install crystal4D before proceeding"
+        )
 
     assert len(Rx) == len(Ry)
     peaks = []
@@ -435,8 +436,10 @@ def find_Bragg_disks_aiml_serial(
 
     try:
         import crystal4D
-    except:
-        raise ImportError("Import Error: Please install crystal4D before proceeding")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "Import Error: Please install crystal4D before proceeding"
+        )
 
     # Make the peaks PointListArray
     dtype = [("qx", float), ("qy", float), ("intensity", float)]
@@ -645,8 +648,10 @@ def find_Bragg_disks_aiml(
     """
     try:
         import crystal4D
-    except:
-        raise ImportError("Please install crystal4D before proceeding")
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
+            "Import Error: Please install crystal4D before proceeding"
+        )
 
     def _parse_distributed(distributed):
         import os
@@ -842,7 +847,8 @@ def _integrate_disks(DP, maxima_x, maxima_y, maxima_int, int_window_radius=1):
         disks.append(np.average(disk))
     try:
         disks = disks / max(disks)
-    except:
+    # possibly a ZeroDivideError
+    except Exception:
         pass
     return (maxima_x, maxima_y, disks)
 
@@ -880,8 +886,8 @@ def _get_latest_model(model_path=None):
 
     try:
         import tensorflow as tf
-    except:
-        raise ImportError(
+    except ModuleNotFoundError:
+        raise ModuleNotFoundError(
             "Please install tensorflow before proceeding - please check "
             + "https://www.tensorflow.org/install"
             + "for more information"
@@ -893,8 +899,11 @@ def _get_latest_model(model_path=None):
     if model_path is None:
         try:
             os.mkdir("./tmp")
-        except:
+        except FileExistsError:
+            pass
+        except Exception as e:
             pass
+            # raise e
         # download the json file with the meta data
         gdrive_download(
             "FCU-Net",
@@ -912,7 +921,8 @@ def _get_latest_model(model_path=None):
             with open("./tmp/model_metadata_old.json") as f_old:
                 metaold = json.load(f_old)
                 file_id_old = metaold["file_id"]
-        except:
+        # I think just FileNotFoundError
+        except (FileNotFoundError, Exception):
             file_id_old = file_id
 
         if os.path.exists(file_path) and file_id == file_id_old:
@@ -929,7 +939,7 @@ def _get_latest_model(model_path=None):
             gdrive_download(file_id, destination="./tmp", filename=filename.name)
             try:
                 shutil.unpack_archive(filename, "./tmp", format="zip")
-            except:
+            except Exception:
                 pass
             model_path = file_path
             os.rename("./tmp/model_metadata.json", "./tmp/model_metadata_old.json")

py4DSTEM/braggvectors/diskdetection_aiml_cuda.py

@@ -9,7 +9,6 @@
 from emdfile import tqdmnd
 from py4DSTEM.braggvectors.braggvectors import BraggVectors
 from emdfile import PointList, PointListArray
-from py4DSTEM.data import QPoints
 from py4DSTEM.braggvectors.kernels import kernels
 from py4DSTEM.braggvectors.diskdetection_aiml import _get_latest_model
 
@@ -23,8 +22,8 @@
 
 try:
     import tensorflow as tf
-except:
-    raise ImportError(
+except ModuleNotFoundError:
+    raise ModuleNotFoundError(
         "Please install tensorflow before proceeding - please check "
         + "https://www.tensorflow.org/install"
         + "for more information"
@@ -637,7 +636,8 @@ def upsampled_correlation_cp(imageCorr, upsampleFactor, xyShift):
         )
         dx = (icc[2, 1] - icc[0, 1]) / (4 * icc[1, 1] - 2 * icc[2, 1] - 2 * icc[0, 1])
         dy = (icc[1, 2] - icc[1, 0]) / (4 * icc[1, 1] - 2 * icc[1, 2] - 2 * icc[1, 0])
-    except:
+    # I think this is just the IndexError
+    except Exception:
         dx, dy = (
             0,
             0,
@@ -733,6 +733,7 @@ def _integrate_disks_cp(DP, maxima_x, maxima_y, maxima_int, int_window_radius=1)
         disks.append(np.average(disk))
     try:
         disks = disks / max(disks)
-    except:
+    # Possibly ZeroDivisionError
+    except Exception:
         pass
     return (maxima_x, maxima_y, disks)

py4DSTEM/braggvectors/diskdetection_cuda.py

@@ -621,7 +621,8 @@ def upsampled_correlation(imageCorr, upsampleFactor, xyShift):
             dy = (icc[1, 2] - icc[1, 0]) / (
                 4 * icc[1, 1] - 2 * icc[1, 2] - 2 * icc[1, 0]
             )
-        except:
+        # TODO Work out what exception to use  IndexError
+        except Exception:
             dx, dy = (
                 0,
                 0,

py4DSTEM/braggvectors/diskdetection_parallel_new.py

@@ -1,24 +1,16 @@
 import numpy as np
-import matplotlib.pyplot as plt
-import h5py
-import time
 import dill
 
 import dask
 import dask.array as da
 import dask.config
 from dask import delayed
 from dask.distributed import Client, LocalCluster
-from dask.diagnostics import ProgressBar
 
-# import dask.bag as db
-
-# import distributed
 from distributed.protocol.serialize import register_serialization_family
 import distributed
 
-import py4DSTEM
-from emdfile import PointListArray, PointList
+from emdfile import PointListArray
 from py4DSTEM.braggvectors.diskdetection import _find_Bragg_disks_single_DP_FK
 
 

py4DSTEM/braggvectors/threshold.py

@@ -25,7 +25,7 @@ def threshold_Braggpeaks(
             pattern
     """
     assert all(
-        [item in pointlistarray.dtype.fields for item in ["qx", "qy", "intensity"]]
+        item in pointlistarray.dtype.fields for item in ["qx", "qy", "intensity"]
     ), "pointlistarray must include the coordinates 'qx', 'qy', and 'intensity'."
     for Rx, Ry in tqdmnd(
         pointlistarray.shape[0],
@@ -112,7 +112,7 @@ def universal_threshold(
     assert isinstance(pointlistarray, PointListArray)
     assert metric in ("maximum", "average", "median", "manual")
     assert all(
-        [item in pointlistarray.dtype.fields for item in ["qx", "qy", "intensity"]]
+        item in pointlistarray.dtype.fields for item in ["qx", "qy", "intensity"]
     ), "pointlistarray must include the coordinates 'qx', 'qy', and 'intensity'."
     _pointlistarray = pointlistarray.copy()
     if name is None:

py4DSTEM/data/calibration.py

@@ -1,9 +1,7 @@
 # Defines the Calibration class, which stores calibration metadata
 
 import numpy as np
-from numbers import Number
 from typing import Optional
-from warnings import warn
 
 from emdfile import Metadata, Root
 from py4DSTEM.data.propagating_calibration import call_calibrate
@@ -505,7 +503,7 @@ def get_origin(self, rx=None, ry=None):
         qx0 = self._get_value("qx0", rx, ry)
         qy0 = self._get_value("qy0", rx, ry)
         ans = (qx0, qy0)
-        if any([x is None for x in ans]):
+        if any(x is None for x in ans):
             ans = None
         return ans
 
@@ -518,7 +516,7 @@ def get_origin_shift(self, rx=None, ry=None):
         qx0 = self._get_value("qx0_shift", rx, ry)
         qy0 = self._get_value("qy0_shift", rx, ry)
         ans = (qx0, qy0)
-        if any([x is None for x in ans]):
+        if any(x is None for x in ans):
             ans = None
         return ans
 
@@ -540,7 +538,7 @@ def get_origin_meas(self, rx=None, ry=None):
         qx0 = self._get_value("qx0_meas", rx, ry)
         qy0 = self._get_value("qy0_meas", rx, ry)
         ans = (qx0, qy0)
-        if any([x is None for x in ans]):
+        if any(x is None for x in ans):
             ans = None
         return ans
 
@@ -615,7 +613,7 @@ def get_ellipse(self, rx=None, ry=None):
         b = self.get_b(rx, ry)
         theta = self.get_theta(rx, ry)
         ans = (a, b, theta)
-        if any([x is None for x in ans]):
+        if any(x is None for x in ans):
             ans = None
         return ans
 
@@ -778,7 +776,7 @@ def get_probe_param(self):
         qx0 = self._get_value("qx0")
         qy0 = self._get_value("qy0")
         ans = (probe_semiangle, qx0, qy0)
-        if any([x is None for x in ans]):
+        if any(x is None for x in ans):
             ans = None
         return ans