Batch Edit, Workbench, Query Builder backend improvements

.env

@@ -1,3 +1,4 @@
+
 DATABASE_HOST=mariadb
 DATABASE_PORT=3306
 MYSQL_ROOT_PASSWORD=password
@@ -39,4 +40,4 @@ LOG_LEVEL=WARNING
 # should only be used during development and troubleshooting and not
 # during general use. Django applications leak memory when operated
 # continuously in debug mode.
-SP7_DEBUG=true
+SP7_DEBUG=true

.github/workflows/docker.yml

@@ -58,7 +58,6 @@ jobs:
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT
           echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-
       - name: Docker meta
         id: meta
         uses: docker/metadata-action@v5
@@ -76,7 +75,6 @@ jobs:
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Build and push by digest
         id: build
         uses: docker/build-push-action@v6
@@ -93,7 +91,6 @@ jobs:
           mkdir -p /tmp/digests
           digest="${{ steps.build.outputs.digest }}"
           touch "/tmp/digests/${digest#sha256:}"
-
       - name: Upload digest
         uses: actions/upload-artifact@v4
         with:
@@ -122,13 +119,11 @@ jobs:
         uses: docker/metadata-action@v5
         with:
           images: ${{ env.REGISTRY_IMAGE }}
-
       - name: Login to Docker Hub
         uses: docker/login-action@v3
         with:
           username: ${{ secrets.DOCKERHUB_USERNAME }}
           password: ${{ secrets.DOCKERHUB_TOKEN }}
-
       - name: Create manifest list and push
         working-directory: /tmp/digests
         run: |

requirements.txt

@@ -9,3 +9,4 @@ PyJWT==2.3.0
 django-auth-ldap==1.2.15
 jsonschema==3.2.0
 typing-extensions==4.3.0
+django-model-utils==4.4.0

specifyweb/businessrules/uniqueness_rules.py

@@ -25,7 +25,6 @@
 
 logger = logging.getLogger(__name__)
 
-
 @orm_signal_handler('pre_save', None, dispatch_uid=UNIQUENESS_DISPATCH_UID)
 def check_unique(model, instance):
     model_name = instance.__class__.__name__

specifyweb/businessrules/views.py

@@ -294,4 +294,4 @@ def validate_uniqueness(request):
         "fields": [{"duplicates": duplicate[duplicates_field], "fields": {field: value for field, value in duplicate.items() if field != duplicates_field}}
                    for duplicate in duplicates]}
 
-    return http.JsonResponse(final, safe=False)
+    return http.JsonResponse(final, safe=False)

specifyweb/export/dwca.py

@@ -12,8 +12,8 @@
 from xml.etree import ElementTree as ET
 from xml.dom import minidom
 
-from specifyweb.stored_queries.execution import EphemeralField, query_to_csv
-from specifyweb.stored_queries.queryfield import QueryField
+from specifyweb.stored_queries.execution import query_to_csv
+from specifyweb.stored_queries.queryfield import QueryField, EphemeralField
 from specifyweb.stored_queries.models import session_context
 
 logger = logging.getLogger(__name__)

specifyweb/express_search/related.py

@@ -5,67 +5,79 @@
 import logging
 
 from ..specify.models import datamodel
-from ..stored_queries.execution import build_query
+from ..stored_queries.execution import BuildQueryProps, build_query
 from ..stored_queries.query_ops import QueryOps
 from ..stored_queries.queryfield import QueryField
 from ..stored_queries.queryfieldspec import QueryFieldSpec
 
 logger = logging.getLogger(__name__)
 
+
 class F(str):
     pass
 
+
 class RelatedSearchMeta(type):
     def __new__(cls, name, bases, dict):
         Rs = super(RelatedSearchMeta, cls).__new__(cls, name, bases, dict)
         if Rs.definitions is None:
             return Rs
 
-        root_table_name = Rs.definitions[0].split('.')[0]
+        root_table_name = Rs.definitions[0].split(".")[0]
 
         Rs.root = datamodel.get_table(root_table_name, strict=True)
 
         def col_to_fs(col, add_id=False):
-            return QueryFieldSpec.from_path( [root_table_name] + col.split('.'), add_id )
+            return QueryFieldSpec.from_path([root_table_name] + col.split("."), add_id)
 
         Rs.display_fields = [
-            QueryField(fieldspec=col_to_fs(col),
-                       op_num=1,
-                       value="",
-                       negate=False,
-                       display=True,
-                       format_name=None,
-                       sort_type=0,
-                       strict=False)
-            for col in Rs.columns]
-
-        if Rs.link:
-            Rs.display_fields.append(QueryField(
-                fieldspec=col_to_fs(Rs.link, add_id=True),
+            QueryField(
+                fieldspec=col_to_fs(col),
                 op_num=1,
                 value="",
                 negate=False,
                 display=True,
                 format_name=None,
                 sort_type=0,
-                strict=False))
+                strict=False
+            )
+            for col in Rs.columns
+        ]
+
+        if Rs.link:
+            Rs.display_fields.append(
+                QueryField(
+                    fieldspec=col_to_fs(Rs.link, add_id=True),
+                    op_num=1,
+                    value="",
+                    negate=False,
+                    display=True,
+                    format_name=None,
+                    sort_type=0,
+                    strict=False
+                )
+            )
 
         def make_filter(f, negate):
             field, op, val = f
-            return QueryField(fieldspec=col_to_fs(field),
-                              op_num=QueryOps.OPERATIONS.index(op.__name__),
-                              value=col_to_fs(val) if isinstance(val, F) else val,
-                              negate=negate,
-                              display=False,
-                              format_name=None,
-                              sort_type=0,
-                              strict=False)
-
-        Rs.filter_fields = [make_filter(f, False) for f in Rs.filters] + \
-                           [make_filter(f, True) for f in Rs.excludes]
+            return QueryField(
+                fieldspec=col_to_fs(field),
+                op_num=QueryOps.OPERATIONS.index(op.__name__),
+                value=col_to_fs(val) if isinstance(val, F) else val,
+                negate=negate,
+                display=False,
+                format_name=None,
+                sort_type=0,
+                strict=False
+            )
+
+        Rs.filter_fields = [make_filter(f, False) for f in Rs.filters] + [
+            make_filter(f, True) for f in Rs.excludes
+        ]
 
         return Rs
 
+
 class RelatedSearch(object, metaclass=RelatedSearchMeta):
     distinct = False
     filters = []
@@ -76,9 +88,14 @@ class RelatedSearch(object, metaclass=RelatedSearchMeta):
 
     @classmethod
     def execute(cls, session, config, terms, collection, user, limit, offset):
-        queries = [_f for _f in (
-            cls(defn).build_related_query(session, config, terms, collection, user)
-            for defn in cls.definitions) if _f]
+        queries = [
+            _f
+            for _f in (
+                cls(defn).build_related_query(session, config, terms, collection, user)
+                for defn in cls.definitions
+            )
+            if _f
+        ]
 
         if len(queries) > 0:
             query = queries[0].union(*queries[1:])
@@ -89,64 +106,86 @@ def execute(cls, session, config, terms, collection, user, limit, offset):
             results = []
 
         return {
-            'totalCount': count,
-            'results': results,
-            'definition': {
-                'name': cls.__name__,
-                'root': cls.root.name,
-                'link': cls.link,
-                'columns': cls.columns,
-                'fieldSpecs': [{'stringId': fs.to_stringid(), 'isRelationship': fs.is_relationship()}
-                               for field in cls.display_fields
-                               for fs in [field.fieldspec]]}}
+            "totalCount": count,
+            "results": results,
+            "definition": {
+                "name": cls.__name__,
+                "root": cls.root.name,
+                "link": cls.link,
+                "columns": cls.columns,
+                "fieldSpecs": [
+                    {
+                        "stringId": fs.to_stringid(),
+                        "isRelationship": fs.is_relationship(),
+                    }
+                    for field in cls.display_fields
+                    for fs in [field.fieldspec]
+                ],
+            },
+        }
 
     def __init__(self, definition):
         self.definition = definition
 
     def build_related_query(self, session, config, terms, collection, user):
-        logger.info('%s: building related query using definition: %s',
-                    self.__class__.__name__, self.definition)
+        logger.info(
+            "%s: building related query using definition: %s",
+            self.__class__.__name__,
+            self.definition,
+        )
 
         from .views import build_primary_query
 
-        primary_fieldspec = QueryFieldSpec.from_path(self.definition.split('.'), add_id=True)
+        primary_fieldspec = QueryFieldSpec.from_path(
+            self.definition.split("."), add_id=True
+        )
 
         pivot = primary_fieldspec.table
 
-        logger.debug('pivoting on: %s', pivot)
-        for searchtable in config.findall('tables/searchtable'):
-            if searchtable.find('tableName').text == pivot.name:
+        logger.debug("pivoting on: %s", pivot)
+        for searchtable in config.findall("tables/searchtable"):
+            if searchtable.find("tableName").text == pivot.name:
                 break
         else:
             return None
 
-        logger.debug('using %s for primary search', searchtable.find('tableName').text)
-        primary_query = build_primary_query(session, searchtable, terms, collection, user, as_scalar=True)
+        logger.debug("using %s for primary search", searchtable.find("tableName").text)
+        primary_query = build_primary_query(
+            session, searchtable, terms, collection, user, as_scalar=True
+        )
 
         if primary_query is None:
             return None
-        logger.debug('primary query: %s', primary_query)
+        logger.debug("primary query: %s", primary_query)
 
         primary_field = QueryField(
             fieldspec=primary_fieldspec,
-            op_num=QueryOps.OPERATIONS.index('op_in'),
+            op_num=QueryOps.OPERATIONS.index("op_in"),
             value=primary_query,
             negate=False,
             display=False,
             format_name=None,
             sort_type=0,
-            strict=False)
+            strict=False
+        )
 
         logger.debug("primary queryfield: %s", primary_field)
         logger.debug("display queryfields: %s", self.display_fields)
         logger.debug("filter queryfields: %s", self.filter_fields)
 
         queryfields = self.display_fields + self.filter_fields + [primary_field]
 
-        related_query, _ = build_query(session, collection, user, self.root.tableId, queryfields, implicit_or=False)
+        related_query, _ = build_query(
+            session,
+            collection,
+            user,
+            self.root.tableId,
+            queryfields,
+            props=BuildQueryProps(implicit_or=True),
+        )
 
         if self.distinct:
             related_query = related_query.distinct()
 
-        logger.debug('related query: %s', related_query)
+        logger.debug("related query: %s", related_query)
         return related_query

specifyweb/frontend/js_src/css/main.css

@@ -255,10 +255,16 @@
     --invalid-cell: theme('colors.red.300');
     --modified-cell: theme('colors.yellow.250');
     --search-result: theme('colors.green.300');
-    @apply dark:[--invalid-cell:theme('colors.red.900')]
+    --updated-cell: theme('colors.cyan.200');
+    --deleted-cell: theme('colors.amber.500');
+    --matched-and-changed-cell: theme('colors.blue.200');
+    @apply dark:[--deleted-cell:theme('colors.amber.600')]
+      dark:[--invalid-cell:theme('colors.red.900')]
+      dark:[--matched-and-changed-cell:theme('colors.fuchsia.700')]
       dark:[--modified-cell:theme('colors.yellow.900')]
       dark:[--new-cell:theme('colors.indigo.900')]
-      dark:[--search-result:theme('colors.green.900')];
+      dark:[--search-result:theme('colors.green.900')]
+      dark:[--updated-cell:theme('colors.cyan.800')];
   }
 
   .custom-select {

specifyweb/frontend/js_src/css/workbench.css

@@ -38,7 +38,10 @@
 }
 
 /* CONTENT styles */
-.wbs-form.wb-show-upload-results .wb-no-match-cell,
+.wbs-form.wb-show-upload-results .wb-no-match-cell
+.wbs-form.wb-show-upload-results .wb-updated-cell
+.wbs-form.wb-show-upload-results .wb-deleted-cell
+.wbs-form.wb-show-upload-results .wb-matched-and-changed-cell
 .wbs-form.wb-focus-coordinates .wb-coordinate-cell {
   @apply text-black dark:text-white;
 }
@@ -50,18 +53,28 @@
 /* Cell navigation */
 
 .wbs-form
-  :is(
-    .wb-no-match-cell,
-    .wb-modified-cell,
-    .htCommentCell,
-    .wb-search-match-cell
-  ),
+  :is(.wb-no-match-cell, .wb-modified-cell, .htCommentCell, .wb-search-match-cell, .wb-updated-cell, .wb-deleted-cell, .wb-matched-and-changed-cell),
 .wb-navigation-section {
   @apply !bg-[color:var(--accent-color)];
 }
 
 /* The order here determines the priority of the states
  * From the lowest till the highest */
+.wbs-form:not(.wb-hide-new-cells) .wb-updated-cell,
+.wb-navigation-section[data-navigation-type='updatedCells'] {
+  --accent-color: var(--updated-cell);
+}
+
+.wbs-form:not(.wb-hide-new-cells) .wb-deleted-cell,
+.wb-navigation-section[data-navigation-type='deletedCells'] {
+  --accent-color: var(--deleted-cell);
+}
+
+.wbs-form:not(.wb-hide-new-cells) .wb-matched-and-changed-cell,
+.wb-navigation-section[data-navigation-type='matchedAndChangedCells'] {
+  --accent-color: var(--matched-and-changed-cell);
+}
+
 .wbs-form:not(.wb-hide-new-cells) .wb-no-match-cell,
 .wb-navigation-section[data-navigation-type='newCells'] {
   --accent-color: var(--new-cell);