Merge branch 'master' into dev-1150-underscores

Author: ethho

.github/workflows/development.yaml

@@ -90,13 +90,28 @@ jobs:
         run: |
           export HOST_UID=$(id -u)
           docker-compose -f LNX-docker-compose.yml up --build --exit-code-from app
+  lint:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        py_ver: ["3.11"]
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{matrix.py_ver}}
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{matrix.py_ver}}
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install flake8 black==24.2.0
+      - name: Run syntax tests
+        run: flake8 datajoint --count --select=E9,F63,F7,F82 --show-source --statistics
       - name: Run style tests
         run: |
           flake8 --ignore=E203,E722,W503 datajoint \
                  --count --max-complexity=62 --max-line-length=127 --statistics
-          black datajoint --check -v
-          black tests --check -v
-          black tests_old --check -v
+          black --required-version '24.2.0' --check -v datajoint tests tests_old
   codespell:
     name: Check for spelling errors
     permissions:

datajoint/autopopulate.py

@@ -1,4 +1,5 @@
 """This module defines class dj.AutoPopulate"""
+
 import logging
 import datetime
 import traceback

datajoint/blob.py

@@ -322,9 +322,11 @@ def pack_recarray(self, array):
             + "\0".join(array.dtype.names).encode()  # number of fields
             + b"\0"
             + b"".join(  # field names
-                self.pack_recarray(array[f])
-                if array[f].dtype.fields
-                else self.pack_array(array[f])
+                (
+                    self.pack_recarray(array[f])
+                    if array[f].dtype.fields
+                    else self.pack_array(array[f])
+                )
                 for f in array.dtype.names
             )
         )

datajoint/connection.py

@@ -2,6 +2,7 @@
 This module contains the Connection class that manages the connection to the database, and
 the ``conn`` function that provides access to a persistent connection in datajoint.
 """
+
 import warnings
 from contextlib import contextmanager
 import pymysql as client

datajoint/declare.py

@@ -2,6 +2,7 @@
 This module hosts functions to convert DataJoint table definitions into mysql table definitions, and to
 declare the corresponding mysql tables.
 """
+
 import re
 import pyparsing as pp
 import logging
@@ -382,9 +383,7 @@ def _make_attribute_alter(new, old, primary_key):
                         command=(
                             "ADD"
                             if (old_name or new_name) not in old_names
-                            else "MODIFY"
-                            if not old_name
-                            else "CHANGE `%s`" % old_name
+                            else "MODIFY" if not old_name else "CHANGE `%s`" % old_name
                         ),
                         new_def=new_def,
                         after="" if after is None else "AFTER `%s`" % after,

datajoint/diagram.py

@@ -385,11 +385,15 @@ def make_dot(self):
                     assert issubclass(cls, Table)
                     description = cls().describe(context=self.context).split("\n")
                     description = (
-                        "-" * 30
-                        if q.startswith("---")
-                        else q.replace("->", "→")
-                        if "->" in q
-                        else q.split(":")[0]
+                        (
+                            "-" * 30
+                            if q.startswith("---")
+                            else (
+                                q.replace("->", "→")
+                                if "->" in q
+                                else q.split(":")[0]
+                            )
+                        )
                         for q in description
                         if not q.startswith("#")
                     )

datajoint/expression.py

@@ -100,9 +100,11 @@ def primary_key(self):
 
     def from_clause(self):
         support = (
-            "(" + src.make_sql() + ") as `$%x`" % next(self._subquery_alias_count)
-            if isinstance(src, QueryExpression)
-            else src
+            (
+                "(" + src.make_sql() + ") as `$%x`" % next(self._subquery_alias_count)
+                if isinstance(src, QueryExpression)
+                else src
+            )
             for src in self.support
         )
         clause = next(support)
@@ -704,14 +706,16 @@ def make_sql(self, fields=None):
             fields=fields,
             from_=self.from_clause(),
             where=self.where_clause(),
-            group_by=""
-            if not self.primary_key
-            else (
-                " GROUP BY `%s`" % "`,`".join(self._grouping_attributes)
-                + (
-                    ""
-                    if not self.restriction
-                    else " HAVING (%s)" % ")AND(".join(self.restriction)
+            group_by=(
+                ""
+                if not self.primary_key
+                else (
+                    " GROUP BY `%s`" % "`,`".join(self._grouping_attributes)
+                    + (
+                        ""
+                        if not self.restriction
+                        else " HAVING (%s)" % ")AND(".join(self.restriction)
+                    )
                 )
             ),
         )
@@ -773,12 +777,16 @@ def make_sql(self):
             # no secondary attributes: use UNION DISTINCT
             fields = arg1.primary_key
             return "SELECT * FROM (({sql1}) UNION ({sql2})) as `_u{alias}`".format(
-                sql1=arg1.make_sql()
-                if isinstance(arg1, Union)
-                else arg1.make_sql(fields),
-                sql2=arg2.make_sql()
-                if isinstance(arg2, Union)
-                else arg2.make_sql(fields),
+                sql1=(
+                    arg1.make_sql()
+                    if isinstance(arg1, Union)
+                    else arg1.make_sql(fields)
+                ),
+                sql2=(
+                    arg2.make_sql()
+                    if isinstance(arg2, Union)
+                    else arg2.make_sql(fields)
+                ),
                 alias=next(self.__count),
             )
         # with secondary attributes, use union of left join with antijoin

datajoint/fetch.py

@@ -244,13 +244,15 @@ def __call__(
                 ]
             else:
                 return_values = [
-                    list(
-                        (to_dicts if as_dict else lambda x: x)(
-                            ret[self._expression.primary_key]
+                    (
+                        list(
+                            (to_dicts if as_dict else lambda x: x)(
+                                ret[self._expression.primary_key]
+                            )
                         )
+                        if is_key(attribute)
+                        else ret[attribute]
                     )
-                    if is_key(attribute)
-                    else ret[attribute]
                     for attribute in attrs
                 ]
                 ret = return_values[0] if len(attrs) == 1 else return_values
@@ -272,12 +274,14 @@ def __call__(
                     else np.dtype(
                         [
                             (
-                                name,
-                                type(value),
-                            )  # use the first element to determine blob type
-                            if heading[name].is_blob
-                            and isinstance(value, numbers.Number)
-                            else (name, heading.as_dtype[name])
+                                (
+                                    name,
+                                    type(value),
+                                )  # use the first element to determine blob type
+                                if heading[name].is_blob
+                                and isinstance(value, numbers.Number)
+                                else (name, heading.as_dtype[name])
+                            )
                             for value, name in zip(ret[0], heading.as_dtype.names)
                         ]
                     )
@@ -353,9 +357,11 @@ def __call__(self, *attrs, squeeze=False, download_path="."):
                     "fetch1 should only return one tuple. %d tuples found" % len(result)
                 )
             return_values = tuple(
-                next(to_dicts(result[self._expression.primary_key]))
-                if is_key(attribute)
-                else result[attribute][0]
+                (
+                    next(to_dicts(result[self._expression.primary_key]))
+                    if is_key(attribute)
+                    else result[attribute][0]
+                )
                 for attribute in attrs
             )
             ret = return_values[0] if len(attrs) == 1 else return_values

datajoint/heading.py

@@ -193,10 +193,12 @@ def as_sql(self, fields, include_aliases=True):
         represent heading as the SQL SELECT clause.
         """
         return ",".join(
-            "`%s`" % name
-            if self.attributes[name].attribute_expression is None
-            else self.attributes[name].attribute_expression
-            + (" as `%s`" % name if include_aliases else "")
+            (
+                "`%s`" % name
+                if self.attributes[name].attribute_expression is None
+                else self.attributes[name].attribute_expression
+                + (" as `%s`" % name if include_aliases else "")
+            )
             for name in fields
         )
 
@@ -371,9 +373,11 @@ def _init_from_database(self):
                     is_blob=category in ("INTERNAL_BLOB", "EXTERNAL_BLOB"),
                     uuid=category == "UUID",
                     is_external=category in EXTERNAL_TYPES,
-                    store=attr["type"].split("@")[1]
-                    if category in EXTERNAL_TYPES
-                    else None,
+                    store=(
+                        attr["type"].split("@")[1]
+                        if category in EXTERNAL_TYPES
+                        else None
+                    ),
                 )
 
             if attr["in_key"] and any(

datajoint/preview.py

@@ -126,9 +126,9 @@ def repr_html(query_expression):
             head_template.format(
                 column=c,
                 comment=heading.attributes[c].comment,
-                primary="primary"
-                if c in query_expression.primary_key
-                else "nonprimary",
+                primary=(
+                    "primary" if c in query_expression.primary_key else "nonprimary"
+                ),
             )
             for c in heading.names
         ),
@@ -145,7 +145,9 @@ def repr_html(query_expression):
                 for tup in tuples
             ]
         ),
-        count=("<p>Total: %d</p>" % len(rel))
-        if config["display.show_tuple_count"]
-        else "",
+        count=(
+            ("<p>Total: %d</p>" % len(rel))
+            if config["display.show_tuple_count"]
+            else ""
+        ),
     )

datajoint/s3.py

@@ -1,6 +1,7 @@
 """
 AWS S3 operations
 """
+
 from io import BytesIO
 import minio  # https://docs.minio.io/docs/python-client-api-reference
 import urllib3

datajoint/settings.py

@@ -1,6 +1,7 @@
 """
 Settings for DataJoint.
 """
+
 from contextlib import contextmanager
 import json
 import os

datajoint/table.py

@@ -770,9 +770,11 @@ def describe(self, context=None, printout=False):
             if do_include:
                 attributes_declared.add(attr.name)
                 definition += "%-20s : %-28s %s\n" % (
-                    attr.name
-                    if attr.default is None
-                    else "%s=%s" % (attr.name, attr.default),
+                    (
+                        attr.name
+                        if attr.default is None
+                        else "%s=%s" % (attr.name, attr.default)
+                    ),
                     "%s%s"
                     % (attr.type, " auto_increment" if attr.autoincrement else ""),
                     "# " + attr.comment if attr.comment else "",

tests/schema_simple.py

@@ -1,6 +1,7 @@
 """
 A simple, abstract schema to test relational algebra
 """
+
 import random
 import datajoint as dj
 import itertools

tests/test_utils.py

@@ -1,6 +1,7 @@
 """
 Collection of test cases to test core module.
 """
+
 from datajoint import DataJointError
 from datajoint.utils import from_camel_case, to_camel_case
 import pytest

tests_old/schema_simple.py

@@ -1,6 +1,7 @@
 """
 A simple, abstract schema to test relational algebra
 """
+
 import random
 import datajoint as dj
 import itertools

tests_old/test_utils.py

@@ -1,6 +1,7 @@
 """
 Collection of test cases to test core module.
 """
+
 from nose.tools import assert_true, assert_raises, assert_equal
 from datajoint import DataJointError
 from datajoint.utils import from_camel_case, to_camel_case