From 372b7baccdef50a6e8b35c967166c93c6edf67fb Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 17 Mar 2023 14:27:35 -0500 Subject: [PATCH 01/31] Add default forms for CollectionRelationship and CollectionRelType --- .../js_src/lib/components/FormParse/webOnlyViews.ts | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts b/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts index 159d8c70672..e51df316e4f 100644 --- a/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts +++ b/specifyweb/frontend/js_src/lib/components/FormParse/webOnlyViews.ts @@ -57,6 +57,18 @@ export const webOnlyViews = f.store(() => 'spReports', ]) ), + CollectionRelType: autoGenerateViewDefinition( + schema.models.CollectionRelType, + 'form', + 'edit', + ['name', 'leftSideCollection', 'rightSideCollection', 'remarks'] + ), + CollectionRelationship: autoGenerateViewDefinition( + schema.models.CollectionRelationship, + 'form', + 'edit', + ['collectionRelType', 'leftSide', 'rightSide'] + ), [spAppResourceView]: autoGenerateViewDefinition( schema.models.SpAppResource, 'form', From 5967b2027a49c47b6bac9c204d8bd90001c03088 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 17 Mar 2023 14:28:25 -0500 Subject: [PATCH 02/31] Add schema overrides for CollectionRelationship and CollectionRelType --- .../js_src/lib/components/DataModel/schemaOverrides.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts b/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts index 9872f16e56b..bce230c54b9 100644 --- a/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts +++ b/specifyweb/frontend/js_src/lib/components/DataModel/schemaOverrides.ts @@ -110,6 +110,12 @@ const globalFieldOverrides: { Attachment: { tableID: 'optional', }, + CollectionRelationship: { + collectionRelType: 'required', + }, + CollectionRelType: { + name: 'required', + }, Taxon: { parent: 'required', isAccepted: 'readOnly', From 2679c7ec2383666ccfe05625fa8707170b787653 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 17 Mar 2023 14:29:19 -0500 Subject: [PATCH 03/31] Make CollectionRelType name a front-end only PickList Because this table is required for Collection Relationship and it resolves the left side and right side collections through the RelType name. This makes it so that only the available/valid names appear in the Workbench when uploading Collection Relationships --- .../frontend/js_src/lib/components/PickLists/definitions.ts | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts b/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts index ce2d46d907c..00c29998abb 100644 --- a/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts +++ b/specifyweb/frontend/js_src/lib/components/PickLists/definitions.ts @@ -202,6 +202,12 @@ export const getFrontEndPickLists = f.store<{ .set('tableName', 'preptype') .set('fieldName', 'name'), }, + CollectionRelType: { + name: definePicklist('_CollectionRelType', []) + .set('type', PickListTypes.FIELDS) + .set('tableName', 'collectionreltype') + .set('fieldName', 'name'), + }, SpAppResource: { mimeType: definePicklist( '_MimeType', From bf5b8ce0ee4e89684cd8b77117d3bdae5af2e078 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Tue, 21 Mar 2023 23:26:31 -0500 Subject: [PATCH 04/31] Allow manual override of collection for workbench uploads As a property of any 'uploadTable' in the upload plan, the following can be added to manually set the desired collection to upload records into for that column 'overrideScope' : { 'collection' : } where is the id of the collection. --- specifyweb/workbench/upload/scoping.py | 10 +++++++++- specifyweb/workbench/upload/upload_plan_schema.py | 15 +++++++++++++++ specifyweb/workbench/upload/upload_table.py | 1 + 3 files changed, 25 insertions(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index c3f252d61df..57fe2fdddbe 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, Union, Callable +from typing import Dict, Any, Optional, Callable from specifyweb.specify.datamodel import datamodel, Table, Relationship from specifyweb.specify.load_datamodel import DoesNotExistError @@ -80,11 +80,19 @@ def extend_columnoptions(colopts: ColumnOptions, collection, tablename: str, fie dateformat=get_date_format(), ) +def get_scoping_overrides(ut: UploadTable) -> Optional[int]: + if (ut.overrideScope is not None): + return ut.overrideScope def apply_scoping_to_uploadtable(ut: UploadTable, collection) -> ScopedUploadTable: table = datamodel.get_table_strict(ut.name) adjust_to_ones = to_one_adjustments(collection, table) + + scope_overrides = get_scoping_overrides(ut) + if scope_overrides is not None: + collection = models.Collection.objects.filter(id=scope_overrides['collection']).get() + return ScopedUploadTable( name=ut.name, diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 1f05b322828..70cb528d7f8 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -42,6 +42,14 @@ 'type': 'object', 'description': 'The uploadTable structure defines how to upload data for a given table.', 'properties': { + 'overrideScope' : { + 'description' : '', + 'type' : 'object', + 'properties': { + 'collection' : { '$ref': '#/definitions/scopingOverride'} + }, + 'additionalProperties' : False, + }, 'wbcols': { '$ref': '#/definitions/wbcols' }, 'static': { '$ref': '#/definitions/static' }, 'toOne': { '$ref': '#/definitions/toOne' }, @@ -196,6 +204,12 @@ {'ispublic': True, 'license': 'CC BY-NC-ND 2.0'} ] }, + 'scopingOverride' : { + 'description' : '', + 'default' : None, + 'oneOf' : [ {'type': 'integer'}, + {'type': 'null'}] + } } } @@ -232,6 +246,7 @@ def rel_table(key: str) -> Table: return UploadTable( name=table.django_name, + overrideScope= to_parse['overrideScope'] if 'overrideScope' in to_parse.keys() else None, wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, static=to_parse['static'], toOne={ diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 0d20aede145..7d78640bccd 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -21,6 +21,7 @@ class UploadTable(NamedTuple): name: str + overrideScope: Optional[Dict[str, Optional[int]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] From 740bd35a4fc45ac28efb5fe600cd411338b5a6fd Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 22 Mar 2023 17:03:11 -0500 Subject: [PATCH 05/31] Automatically override scope for Collection Relationships --- specifyweb/workbench/upload/upload.py | 22 +++++++++++- .../workbench/upload/upload_plan_schema.py | 34 +++++++++++++++++-- specifyweb/workbench/upload/upload_table.py | 22 ++++++++++++ 3 files changed, 74 insertions(+), 4 deletions(-) diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 0a9bf6e2b82..b772f6c2c12 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -11,9 +11,11 @@ from jsonschema import validate # type: ignore from specifyweb.specify import models +from specifyweb.specify.datamodel import datamodel from specifyweb.specify.auditlog import auditlog from specifyweb.specify.datamodel import Table from specifyweb.specify.tree_extras import renumber_tree, reset_fullnames +from specifyweb.workbench.upload.upload_table import DeferredScopeUploadTable from . import disambiguation from .upload_plan_schema import schema, parse_plan_with_basetable from .upload_result import Uploaded, UploadResult, ParseFailures, \ @@ -187,6 +189,23 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab base_table, plan = parse_plan_with_basetable(collection, plan) return base_table, plan.apply_scoping(collection) +def apply_deferred_scopes(default_collection, upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: + _upload_plan = upload_plan + if hasattr(upload_plan, 'toOne'): + for key, uploadable in upload_plan.toOne.items(): + if isinstance(uploadable, DeferredScopeUploadTable): + related_uploadable = upload_plan.toOne[uploadable.related_key] + related_column_name = related_uploadable.wbcols['name'][0] + filter_value = rows[0][related_column_name] + + filter_search = {uploadable.filter_field : filter_value} + related = getattr(models, datamodel.get_table(uploadable.related_key).django_name).objects.get(**filter_search) + collection_id = getattr(related, uploadable.relationship_name).id + scoped = uploadable.add_colleciton_override(collection_id).apply_scoping(default_collection, deffer=False) + _upload_plan.toOne[key] = scoped + return _upload_plan + + def do_upload( collection, @@ -201,6 +220,7 @@ def do_upload( cache: Dict = {} _auditor = Auditor(collection=collection, audit_log=None if no_commit else auditlog) total = len(rows) if isinstance(rows, Sized) else None + deffered_upload_plan = apply_deferred_scopes(collection, upload_plan, rows) with savepoint("main upload"): tic = time.perf_counter() results: List[UploadResult] = [] @@ -208,7 +228,7 @@ def do_upload( _cache = cache.copy() if cache is not None and allow_partial else cache da = disambiguations[i] if disambiguations else None with savepoint("row upload") if allow_partial else no_savepoint(): - bind_result = upload_plan.disambiguate(da).bind(collection, row, uploading_agent_id, _auditor, cache) + bind_result = deffered_upload_plan.disambiguate(da).bind(collection, row, uploading_agent_id, _auditor, cache) result = UploadResult(bind_result, {}, {}) if isinstance(bind_result, ParseFailures) else bind_result.process_row() results.append(result) if progress is not None: diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 70cb528d7f8..9aca62369e9 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -4,7 +4,7 @@ from specifyweb.specify.load_datamodel import DoesNotExistError from specifyweb.specify import models -from .upload_table import UploadTable, OneToOneTable, MustMatchTable +from .upload_table import DeferredScopeUploadTable, UploadTable, OneToOneTable, MustMatchTable from .tomany import ToManyRecord from .treerecord import TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable @@ -241,8 +241,34 @@ def parse_uploadable(collection, table: Table, to_parse: Dict) -> Uploadable: def parse_upload_table(collection, table: Table, to_parse: Dict) -> UploadTable: + defer_scope = {("Collectionrelationship", "rightside"): ('collectionreltype', 'name')} + def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) + + def defer_scope_upload_table(default_collection, table: Table, to_parse:Dict, deferred_information) -> DeferredScopeUploadTable: + related_key = defer_scope[deferred_information][0] + filter_field = defer_scope[deferred_information][1] + relationship_overrides = {"rightside" : "rightsidecollection"} + relationship_name = deferred_information[1] if deferred_information[1] not in relationship_overrides.keys() else relationship_overrides[deferred_information[1]] + return DeferredScopeUploadTable( + name=table.django_name, + related_key=related_key, + relationship_name=relationship_name, + filter_field=filter_field, + overrideScope= to_parse['overrideScope'] if 'overrideScope' in to_parse.keys() else None, + wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, + static=to_parse['static'], + toOne={ + key: defer_scope_upload_table(collection, rel_table(key), to_one, (table.django_name, key)) if (table.django_name, key) in defer_scope.keys() + else parse_uploadable(collection, rel_table(key), to_one) + for key, to_one in to_parse['toOne'].items() + }, + toMany={ + key: [parse_to_many_record(default_collection, rel_table(key), record) for record in to_manys] + for key, to_manys in to_parse['toMany'].items() + } + ) return UploadTable( name=table.django_name, @@ -250,8 +276,10 @@ def rel_table(key: str) -> Table: wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, static=to_parse['static'], toOne={ - key: parse_uploadable(collection, rel_table(key), to_one) - for key, to_one in to_parse['toOne'].items() + key: defer_scope_upload_table(collection, rel_table(key), to_one['uploadTable'], (table.django_name, key)) + if (table.django_name, key) in defer_scope.keys() + else parse_uploadable(collection, rel_table(key), to_one) + for key, to_one in to_parse['toOne'].items() }, toMany={ key: [parse_to_many_record(collection, rel_table(key), record) for record in to_manys] diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 7d78640bccd..b5d53084692 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -56,6 +56,28 @@ def to_json(self) -> Dict: def unparse(self) -> Dict: return { 'baseTableName': self.name, 'uploadable': self.to_json() } + +class DeferredScopeUploadTable(NamedTuple): + name: str + overrideScope: Optional[Dict[str, Optional[int]]] + wbcols: Dict[str, ColumnOptions] + static: Dict[str, Any] + toOne: Dict[str, Uploadable] + toMany: Dict[str, List[ToManyRecord]] + + related_key: str + relationship_name: str + filter_field: str + + def apply_scoping(self, collection, deffer: bool = True) -> "ScopedUploadTable": + if not deffer: + from .scoping import apply_scoping_to_uploadtable as apply_scoping + return apply_scoping(self, collection) + else: return self + + def add_colleciton_override(self, collection_id) -> "DeferredScopeUploadTable": + return self._replace(overrideScope = {"collection": collection_id}) + class ScopedUploadTable(NamedTuple): name: str From 184274de79faf54857afccb234f40596d632732b Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 24 Mar 2023 09:38:24 -0500 Subject: [PATCH 06/31] Apply deferred scoping on a per-row basis --- specifyweb/workbench/upload/scoping.py | 5 +-- specifyweb/workbench/upload/tomany.py | 4 +- specifyweb/workbench/upload/treerecord.py | 6 +-- specifyweb/workbench/upload/upload.py | 28 ++++++------ specifyweb/workbench/upload/upload_table.py | 47 ++++++++++++++++----- specifyweb/workbench/upload/uploadable.py | 2 +- 6 files changed, 60 insertions(+), 32 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 57fe2fdddbe..1a0e2e563ff 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -89,9 +89,8 @@ def apply_scoping_to_uploadtable(ut: UploadTable, collection) -> ScopedUploadTab adjust_to_ones = to_one_adjustments(collection, table) - scope_overrides = get_scoping_overrides(ut) - if scope_overrides is not None: - collection = models.Collection.objects.filter(id=scope_overrides['collection']).get() + if ut.overrideScope is not None and isinstance(ut.overrideScope['collection'], int): + collection = models.Collection.objects.filter(id=ut.overrideScope['collection']).get() return ScopedUploadTable( diff --git a/specifyweb/workbench/upload/tomany.py b/specifyweb/workbench/upload/tomany.py index 36b0bcc3c03..7b08ef53180 100644 --- a/specifyweb/workbench/upload/tomany.py +++ b/specifyweb/workbench/upload/tomany.py @@ -56,12 +56,12 @@ def disambiguate(self, disambiguation: Disambiguation) -> "ScopedToManyRecord": def get_treedefs(self) -> Set: return set(td for toOne in self.toOne.values() for td in toOne.get_treedefs()) - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]) -> Union["BoundToManyRecord", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict], row_index: Optional[int] = None) -> Union["BoundToManyRecord", ParseFailures]: parsedFields, parseFails = parse_many(collection, self.name, self.wbcols, row) toOne: Dict[str, BoundUploadable] = {} for fieldname, uploadable in self.toOne.items(): - result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache) + result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache, row_index) if isinstance(result, ParseFailures): parseFails += result.failures else: diff --git a/specifyweb/workbench/upload/treerecord.py b/specifyweb/workbench/upload/treerecord.py index 660ed31ae9b..14fc286bdab 100644 --- a/specifyweb/workbench/upload/treerecord.py +++ b/specifyweb/workbench/upload/treerecord.py @@ -57,7 +57,7 @@ def disambiguate(self, disambiguation: DA) -> "ScopedTreeRecord": def get_treedefs(self) -> Set: return set([self.treedef]) - def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundTreeRecord", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundTreeRecord", ParseFailures]: parsedFields: Dict[str, List[ParseResult]] = {} parseFails: List[ParseFailure] = [] for rank, cols in self.ranks.items(): @@ -94,8 +94,8 @@ def apply_scoping(self, collection) -> "ScopedMustMatchTreeRecord": return ScopedMustMatchTreeRecord(*s) class ScopedMustMatchTreeRecord(ScopedTreeRecord): - def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundMustMatchTreeRecord", ParseFailures]: - b = super().bind(collection, row, uploadingAgentId, auditor, cache) + def bind(self, collection, row: Row, uploadingAgentId: Optional[int], auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundMustMatchTreeRecord", ParseFailures]: + b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return b if isinstance(b, ParseFailures) else BoundMustMatchTreeRecord(*b) class TreeDefItemWithParseResults(NamedTuple): diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index b772f6c2c12..787f8d35607 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -189,24 +189,28 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab base_table, plan = parse_plan_with_basetable(collection, plan) return base_table, plan.apply_scoping(collection) -def apply_deferred_scopes(default_collection, upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: +def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: _upload_plan = upload_plan if hasattr(upload_plan, 'toOne'): for key, uploadable in upload_plan.toOne.items(): if isinstance(uploadable, DeferredScopeUploadTable): - related_uploadable = upload_plan.toOne[uploadable.related_key] - related_column_name = related_uploadable.wbcols['name'][0] - filter_value = rows[0][related_column_name] - - filter_search = {uploadable.filter_field : filter_value} - related = getattr(models, datamodel.get_table(uploadable.related_key).django_name).objects.get(**filter_search) - collection_id = getattr(related, uploadable.relationship_name).id - scoped = uploadable.add_colleciton_override(collection_id).apply_scoping(default_collection, deffer=False) + + def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int): + related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] + related_column_name = related_uploadable.wbcols['name'][0] + filter_value = rows[row_index][related_column_name] + + filter_search = {deferred_upload_plan.filter_field : filter_value} + related = getattr(models, datamodel.get_table(deferred_upload_plan.related_key).django_name).objects.get(**filter_search) + collection_id = getattr(related, deferred_upload_plan.relationship_name).id + collection = models.Collection.objects.get(id=collection_id) + return collection + + scoped = uploadable.add_colleciton_override(collection_override_function) _upload_plan.toOne[key] = scoped return _upload_plan - def do_upload( collection, rows: Rows, @@ -220,7 +224,7 @@ def do_upload( cache: Dict = {} _auditor = Auditor(collection=collection, audit_log=None if no_commit else auditlog) total = len(rows) if isinstance(rows, Sized) else None - deffered_upload_plan = apply_deferred_scopes(collection, upload_plan, rows) + deffered_upload_plan = apply_deferred_scopes(upload_plan, rows) with savepoint("main upload"): tic = time.perf_counter() results: List[UploadResult] = [] @@ -228,7 +232,7 @@ def do_upload( _cache = cache.copy() if cache is not None and allow_partial else cache da = disambiguations[i] if disambiguations else None with savepoint("row upload") if allow_partial else no_savepoint(): - bind_result = deffered_upload_plan.disambiguate(da).bind(collection, row, uploading_agent_id, _auditor, cache) + bind_result = deffered_upload_plan.disambiguate(da).bind(collection, row, uploading_agent_id, _auditor, cache, i) result = UploadResult(bind_result, {}, {}) if isinstance(bind_result, ParseFailures) else bind_result.process_row() results.append(result) if progress is not None: diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index b5d53084692..376f47a0b5b 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -1,7 +1,7 @@ import logging from functools import reduce -from typing import List, Dict, Any, NamedTuple, Union, Optional, Set +from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable from django.db import transaction, IntegrityError @@ -59,7 +59,7 @@ def unparse(self) -> Dict: class DeferredScopeUploadTable(NamedTuple): name: str - overrideScope: Optional[Dict[str, Optional[int]]] + overrideScope: Optional[Dict[str, Optional[Union[int, Callable[[int], int]]]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] @@ -75,8 +75,33 @@ def apply_scoping(self, collection, deffer: bool = True) -> "ScopedUploadTable": return apply_scoping(self, collection) else: return self - def add_colleciton_override(self, collection_id) -> "DeferredScopeUploadTable": - return self._replace(overrideScope = {"collection": collection_id}) + def add_colleciton_override(self, collection: Union[int, Callable[[int], int]]) -> "DeferredScopeUploadTable": + return self._replace(overrideScope = {"collection": collection}) + + def disambiguate(self, da: Disambiguation): + return self._replace(da = da) + + def get_treedefs(self) -> Set: + return ( + set(td for toOne in self.toOne.values() for td in toOne.get_treedefs()) | + set(td for toMany in self.toMany.values() for tmr in toMany for td in tmr.get_treedefs()) + ) + + def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundUploadTable", ParseFailures]: + if 'collection' in self.overrideScope.keys(): + if isinstance(self.overrideScope['collection'], int): + collection_id = self.overrideScope['collection'] + collection = models.Collection.objects.get(id=collection_id) + scoped = self.apply_scoping(collection, deffer=False) + elif isinstance(self.overrideScope['collection'], Callable): + collection = self.overrideScope['collection'](self, row_index) + scoped = self.apply_scoping(collection, deffer=False) + + if scoped is None: scoped = self.apply_scoping(default_collection, deffer=False) + + scoped_disambiguated = scoped.disambiguate(self.da) if hasattr(self, "da") else scoped + + return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) class ScopedUploadTable(NamedTuple): @@ -114,12 +139,12 @@ def get_treedefs(self) -> Set: ) - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundUploadTable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundUploadTable", ParseFailures]: parsedFields, parseFails = parse_many(collection, self.name, self.wbcols, row) toOne: Dict[str, BoundUploadable] = {} for fieldname, uploadable in self.toOne.items(): - result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache) + result = uploadable.bind(collection, row, uploadingAgentId, auditor, cache, row_index) if isinstance(result, ParseFailures): parseFails += result.failures else: @@ -129,7 +154,7 @@ def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, ca for fieldname, records in self.toMany.items(): boundRecords: List[BoundToManyRecord] = [] for record in records: - result_ = record.bind(collection, row, uploadingAgentId, auditor, cache) + result_ = record.bind(collection, row, uploadingAgentId, auditor, cache, row_index) if isinstance(result_, ParseFailures): parseFails += result_.failures else: @@ -161,8 +186,8 @@ def to_json(self) -> Dict: return { 'oneToOneTable': self._to_json() } class ScopedOneToOneTable(ScopedUploadTable): - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundOneToOneTable", ParseFailures]: - b = super().bind(collection, row, uploadingAgentId, auditor, cache) + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundOneToOneTable", ParseFailures]: + b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return BoundOneToOneTable(*b) if isinstance(b, BoundUploadTable) else b class MustMatchTable(UploadTable): @@ -174,8 +199,8 @@ def to_json(self) -> Dict: return { 'mustMatchTable': self._to_json() } class ScopedMustMatchTable(ScopedUploadTable): - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundMustMatchTable", ParseFailures]: - b = super().bind(collection, row, uploadingAgentId, auditor, cache) + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundMustMatchTable", ParseFailures]: + b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return BoundMustMatchTable(*b) if isinstance(b, BoundUploadTable) else b diff --git a/specifyweb/workbench/upload/uploadable.py b/specifyweb/workbench/upload/uploadable.py index 4e20f563ba2..17045aa6013 100644 --- a/specifyweb/workbench/upload/uploadable.py +++ b/specifyweb/workbench/upload/uploadable.py @@ -39,7 +39,7 @@ class ScopedUploadable(Protocol): def disambiguate(self, disambiguation: Disambiguation) -> "ScopedUploadable": ... - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None) -> Union["BoundUploadable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundUploadable", ParseFailures]: ... def get_treedefs(self) -> Set: From 047b6a407a52598ec1aa247f665dde8f0ca1b412 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 24 Mar 2023 15:31:31 -0500 Subject: [PATCH 07/31] Add code documentation --- specifyweb/workbench/upload/scoping.py | 26 +++++- specifyweb/workbench/upload/upload.py | 37 ++++---- .../workbench/upload/upload_plan_schema.py | 18 ++-- specifyweb/workbench/upload/upload_table.py | 88 +++++++++++++++---- 4 files changed, 123 insertions(+), 46 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 1a0e2e563ff..4df523e8413 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, Optional, Callable +from typing import Dict, Any, Optional, Tuple, Callable from specifyweb.specify.datamodel import datamodel, Table, Relationship from specifyweb.specify.load_datamodel import DoesNotExistError @@ -12,6 +12,30 @@ from .treerecord import TreeRecord, ScopedTreeRecord from .column_options import ColumnOptions, ExtendedColumnOptions +""" There are cases in which the scoping of records should be dependent on another record/column in a WorkBench dataset. + +The DEFERRED_SCOPING dictonary defines the information needed to extract the correct scope to upload/validate a record into. + +The structure of DEFERRED_SCOPING is as following: + The keys are tuples containing the django table name and a relationship that should be scoped. + + The values are tuples containing the table name, field to filter on, and value to pull from that field to use as the collection for the + tableName.fieldName in the associated key of DEFERRED_SCOPING + + For example, consider the following the deferred scoping information: + ("Collectionrelationship", "rightside"): ('collectionreltype', 'name', 'rightsidecollection') + + This information describes the following process to be performed: + + 'when uploading the rightside of a Collection Relationship, get the Collection Rel Type in the database from the dataset by + filtering Collection Rel Types in the database by name. Then, set the collection of the Collectionrelationship rightside equal to the Collection Rel Type's + rightSideCollection' + + See .upload_plan_schema.py for how this is used + +""" +DEFERRED_SCOPING: Dict[Tuple[str, str], Tuple[str, str, str]] = {("Collectionrelationship", "rightside"): ('collectionreltype', 'name', 'rightsidecollection')} + def scoping_relationships(collection, table: Table) -> Dict[str, int]: extra_static: Dict[str, int] = {} diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 787f8d35607..5dfc8b5e8a6 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -190,25 +190,28 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab return base_table, plan.apply_scoping(collection) def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: - _upload_plan = upload_plan + is_deferred = lambda uploadable: isinstance(uploadable, DeferredScopeUploadTable) + + def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection: + related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] + related_column_name = related_uploadable.wbcols['name'][0] + filter_value = rows[row_index][related_column_name] + + filter_search = {deferred_upload_plan.filter_field : filter_value} + related = getattr(models, datamodel.get_table(deferred_upload_plan.related_key).django_name).objects.get(**filter_search) + collection_id = getattr(related, deferred_upload_plan.relationship_name).id + collection = models.Collection.objects.get(id=collection_id) + return collection + if hasattr(upload_plan, 'toOne'): for key, uploadable in upload_plan.toOne.items(): - if isinstance(uploadable, DeferredScopeUploadTable): - - def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int): - related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] - related_column_name = related_uploadable.wbcols['name'][0] - filter_value = rows[row_index][related_column_name] - - filter_search = {deferred_upload_plan.filter_field : filter_value} - related = getattr(models, datamodel.get_table(deferred_upload_plan.related_key).django_name).objects.get(**filter_search) - collection_id = getattr(related, deferred_upload_plan.relationship_name).id - collection = models.Collection.objects.get(id=collection_id) - return collection - - scoped = uploadable.add_colleciton_override(collection_override_function) - _upload_plan.toOne[key] = scoped - return _upload_plan + _uploadable = uploadable + if _uploadable.toOne != {}: _uploadable = apply_deferred_scopes(_uploadable, rows) + if is_deferred(_uploadable): + _uploadable = _uploadable.add_colleciton_override(collection_override_function) + upload_plan.toOne[key] = _uploadable + + return upload_plan def do_upload( diff --git a/specifyweb/workbench/upload/upload_plan_schema.py b/specifyweb/workbench/upload/upload_plan_schema.py index 9aca62369e9..3826fbf4263 100644 --- a/specifyweb/workbench/upload/upload_plan_schema.py +++ b/specifyweb/workbench/upload/upload_plan_schema.py @@ -9,6 +9,7 @@ from .treerecord import TreeRecord, MustMatchTreeRecord from .uploadable import Uploadable from .column_options import ColumnOptions +from .scoping import DEFERRED_SCOPING schema: Dict = { @@ -241,16 +242,14 @@ def parse_uploadable(collection, table: Table, to_parse: Dict) -> Uploadable: def parse_upload_table(collection, table: Table, to_parse: Dict) -> UploadTable: - defer_scope = {("Collectionrelationship", "rightside"): ('collectionreltype', 'name')} - def rel_table(key: str) -> Table: return datamodel.get_table_strict(table.get_relationship(key).relatedModelName) - def defer_scope_upload_table(default_collection, table: Table, to_parse:Dict, deferred_information) -> DeferredScopeUploadTable: - related_key = defer_scope[deferred_information][0] - filter_field = defer_scope[deferred_information][1] - relationship_overrides = {"rightside" : "rightsidecollection"} - relationship_name = deferred_information[1] if deferred_information[1] not in relationship_overrides.keys() else relationship_overrides[deferred_information[1]] + def defer_scope_upload_table(default_collection, table: Table, to_parse: Dict, deferred_information: Tuple[str, str]) -> DeferredScopeUploadTable: + related_key = DEFERRED_SCOPING[deferred_information][0] + filter_field = DEFERRED_SCOPING[deferred_information][1] + relationship_name = DEFERRED_SCOPING[deferred_information][2] + return DeferredScopeUploadTable( name=table.django_name, related_key=related_key, @@ -260,7 +259,8 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse:Dict, de wbcols={k: parse_column_options(v) for k,v in to_parse['wbcols'].items()}, static=to_parse['static'], toOne={ - key: defer_scope_upload_table(collection, rel_table(key), to_one, (table.django_name, key)) if (table.django_name, key) in defer_scope.keys() + key: defer_scope_upload_table(collection, rel_table(key), to_one, (table.django_name, key)) + if (table.django_name, key) in DEFERRED_SCOPING.keys() else parse_uploadable(collection, rel_table(key), to_one) for key, to_one in to_parse['toOne'].items() }, @@ -277,7 +277,7 @@ def defer_scope_upload_table(default_collection, table: Table, to_parse:Dict, de static=to_parse['static'], toOne={ key: defer_scope_upload_table(collection, rel_table(key), to_one['uploadTable'], (table.django_name, key)) - if (table.django_name, key) in defer_scope.keys() + if (table.django_name, key) in DEFERRED_SCOPING.keys() else parse_uploadable(collection, rel_table(key), to_one) for key, to_one in to_parse['toOne'].items() }, diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 376f47a0b5b..d8654bd0921 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -1,7 +1,7 @@ import logging from functools import reduce -from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable +from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal from django.db import transaction, IntegrityError @@ -21,15 +21,15 @@ class UploadTable(NamedTuple): name: str - overrideScope: Optional[Dict[str, Optional[int]]] + overrideScope: Optional[Dict[Literal['collection'], Optional[int]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] toMany: Dict[str, List[ToManyRecord]] def apply_scoping(self, collection) -> "ScopedUploadTable": - from .scoping import apply_scoping_to_uploadtable as apply_scoping - return apply_scoping(self, collection) + from .scoping import apply_scoping_to_uploadtable + return apply_scoping_to_uploadtable(self, collection) def get_cols(self) -> Set[str]: return set(cd.column for cd in self.wbcols.values()) \ @@ -58,8 +58,23 @@ def unparse(self) -> Dict: return { 'baseTableName': self.name, 'uploadable': self.to_json() } class DeferredScopeUploadTable(NamedTuple): + ''' In the case that the scoping of a record in a WorkBench upload can not be known + until the values of the rows are known, the scope of the record should be deferred until + the row is being processed. + + When the upload table is parsed in .upload_plan_schema.py, if a table contains a field which scoping + is unknown, a DeferredScope UploadTable is created. + + As suggested by the name, the DeferredScope UploadTable is not scoped or disambiguated until its bind() + method is called. In which case, the rows of the dataset are known and the scoping can be deduced + ''' name: str - overrideScope: Optional[Dict[str, Optional[Union[int, Callable[[int], int]]]]] + + # In a DeferredScopeUploadTable, the overrideScope value can be either an integer + # (which follows the same logic as in UploadTable), or a function which has the parameter + # signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection + # (see apply_deferred_scopes in .upload.py) + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], models.Collection]]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] @@ -69,17 +84,34 @@ class DeferredScopeUploadTable(NamedTuple): relationship_name: str filter_field: str - def apply_scoping(self, collection, deffer: bool = True) -> "ScopedUploadTable": - if not deffer: - from .scoping import apply_scoping_to_uploadtable as apply_scoping - return apply_scoping(self, collection) + def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", "DeferredScopeUploadTable"]: + if not defer: + from .scoping import apply_scoping_to_uploadtable + return apply_scoping_to_uploadtable(self, collection) else: return self - def add_colleciton_override(self, collection: Union[int, Callable[[int], int]]) -> "DeferredScopeUploadTable": + def add_colleciton_override(self, collection: Union[int, Callable[["DeferredScopeUploadTable", int], models.Collection]]) -> "DeferredScopeUploadTable": + ''' To modify the overrideScope after the DeferredScope UploadTable is created, use add_colleciton_override + To properly apply scoping (see self.bind()), the should either be a collection's id, or a callable (function), + which has paramaters that accept: this DeferredScope UploadTable, and an integer representing the current row_index. + + Note that _replace(**kwargs) does not modify the original object. It insteads creates a new object with the same attributes except for + those added/changed in the paramater kwargs. + + ''' return self._replace(overrideScope = {"collection": collection}) def disambiguate(self, da: Disambiguation): - return self._replace(da = da) + '''Disambiguation should only be used when the UploadTable is completely Scoped. + + When a caller attempts to disambiguate a DeferredScope UploadTable, create and return + a copy of the DeferredScope Upload Table with the Disambiguation stored in a + 'disambiguation' attribute. + + If this attribute exists when the DeferredScoped UploadTable is scoped, + then disambiguate the new Scoped UploadTable using the stored Disambiguation + ''' + return self._replace(disambiguation = da) def get_treedefs(self) -> Set: return ( @@ -87,20 +119,35 @@ def get_treedefs(self) -> Set: set(td for toMany in self.toMany.values() for tmr in toMany for td in tmr.get_treedefs()) ) - def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundUploadTable", ParseFailures]: + def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundUploadTable", ParseFailures]: + + scoped = None + + ''' If the collection should be overridden and an integer (collection id) is provided, + then get the collection with that id and apply the proper scoping. + + Otherwise, if a funciton is provided (see apply_deferred_scopes in .upload.py), then call the function + with the row and row_index to get the needed collection + ''' if 'collection' in self.overrideScope.keys(): if isinstance(self.overrideScope['collection'], int): collection_id = self.overrideScope['collection'] collection = models.Collection.objects.get(id=collection_id) - scoped = self.apply_scoping(collection, deffer=False) + scoped = self.apply_scoping(collection, defer=False) elif isinstance(self.overrideScope['collection'], Callable): collection = self.overrideScope['collection'](self, row_index) - scoped = self.apply_scoping(collection, deffer=False) + scoped = self.apply_scoping(collection, defer=False) - if scoped is None: scoped = self.apply_scoping(default_collection, deffer=False) + # If the collection/scope should not be overriden, defer to the default behavior and assume + # the record should be uploaded in the logged-in collection + if scoped is None: scoped = self.apply_scoping(default_collection, defer=False) - scoped_disambiguated = scoped.disambiguate(self.da) if hasattr(self, "da") else scoped + # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new + # ScopedUploadTable + scoped_disambiguated = scoped.disambiguate(self.da) if hasattr(self, "disambiguation") else scoped + # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) @@ -139,7 +186,8 @@ def get_treedefs(self) -> Set: ) - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundUploadTable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundUploadTable", ParseFailures]: parsedFields, parseFails = parse_many(collection, self.name, self.wbcols, row) toOne: Dict[str, BoundUploadable] = {} @@ -186,7 +234,8 @@ def to_json(self) -> Dict: return { 'oneToOneTable': self._to_json() } class ScopedOneToOneTable(ScopedUploadTable): - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundOneToOneTable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundOneToOneTable", ParseFailures]: b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return BoundOneToOneTable(*b) if isinstance(b, BoundUploadTable) else b @@ -199,7 +248,8 @@ def to_json(self) -> Dict: return { 'mustMatchTable': self._to_json() } class ScopedMustMatchTable(ScopedUploadTable): - def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None) -> Union["BoundMustMatchTable", ParseFailures]: + def bind(self, collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None + ) -> Union["BoundMustMatchTable", ParseFailures]: b = super().bind(collection, row, uploadingAgentId, auditor, cache, row_index) return BoundMustMatchTable(*b) if isinstance(b, BoundUploadTable) else b From ac7e48a01126fed7313d860e9385476174079ffd Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 24 Mar 2023 15:42:04 -0500 Subject: [PATCH 08/31] Add overrideScope in calls to UploadTable in tests --- .../workbench/upload/tests/example_plan.py | 6 ++++++ .../upload/tests/testdisambiguation.py | 3 +++ .../workbench/upload/tests/testparsing.py | 17 ++++++++++++++++ .../workbench/upload/tests/testschema.py | 2 +- .../workbench/upload/tests/testuploading.py | 20 +++++++++++++++++++ 5 files changed, 47 insertions(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/tests/example_plan.py b/specifyweb/workbench/upload/tests/example_plan.py index 2b6381354f2..7aabb873bcb 100644 --- a/specifyweb/workbench/upload/tests/example_plan.py +++ b/specifyweb/workbench/upload/tests/example_plan.py @@ -140,6 +140,7 @@ def with_scoping(collection) -> ScopedUploadTable: wbcols = { 'catalognumber' : parse_column_options("BMSM No."), }, + overrideScope=None, static = {}, toMany = { 'determinations': [ @@ -160,6 +161,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'middleinitial': parse_column_options('Determiner 1 Middle Initial'), 'lastname': parse_column_options('Determiner 1 Last Name'), }, + overrideScope=None, static = {'agenttype': 1}, toOne = {}, toMany = {}, @@ -188,6 +190,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'startdate' : parse_column_options('Start Date Collected'), 'stationfieldnumber' : parse_column_options('Station No.'), }, + overrideScope=None, static = {}, toOne = { 'locality': UploadTable( @@ -197,6 +200,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'latitude1': parse_column_options('Latitude1'), 'longitude1': parse_column_options('Longitude1'), }, + overrideScope=None, static = {'srclatlongunit': 0}, toOne = { 'geography': TreeRecord( @@ -227,6 +231,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'middleinitial' : parse_column_options('Collector 1 Middle Initial'), 'lastname' : parse_column_options('Collector 1 Last Name'), }, + overrideScope=None, static = {'agenttype': 1}, toOne = {}, toMany = {}, @@ -246,6 +251,7 @@ def with_scoping(collection) -> ScopedUploadTable: 'middleinitial' : parse_column_options('Collector 2 Middle Initial'), 'lastname' : parse_column_options('Collector 2 Last name'), }, + overrideScope=None, static = {'agenttype': 1}, toOne = {}, toMany = {}, diff --git a/specifyweb/workbench/upload/tests/testdisambiguation.py b/specifyweb/workbench/upload/tests/testdisambiguation.py index fafa2ea6c72..24ff86b7556 100644 --- a/specifyweb/workbench/upload/tests/testdisambiguation.py +++ b/specifyweb/workbench/upload/tests/testdisambiguation.py @@ -31,6 +31,7 @@ def test_disambiguation(self) -> None: plan = UploadTable( name='Referencework', wbcols={'title': parse_column_options('title')}, + overrideScope=None, static={'referenceworktype': 0}, toOne={}, toMany={'authors': [ @@ -43,6 +44,7 @@ def test_disambiguation(self) -> None: wbcols={ 'lastname': parse_column_options('author1') }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -56,6 +58,7 @@ def test_disambiguation(self) -> None: wbcols={ 'lastname': parse_column_options('author2') }, + overrideScope=None, static={}, toOne={}, toMany={} diff --git a/specifyweb/workbench/upload/tests/testparsing.py b/specifyweb/workbench/upload/tests/testparsing.py index ccbd7fb3189..d8156817035 100644 --- a/specifyweb/workbench/upload/tests/testparsing.py +++ b/specifyweb/workbench/upload/tests/testparsing.py @@ -143,6 +143,7 @@ def test_nonreadonly_picklist(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno'), 'text1': parse_column_options('habitat')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -185,6 +186,7 @@ def test_uiformatter_match(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -210,6 +212,7 @@ def test_numeric_types(self) -> None: 'number1': parse_column_options('float'), 'totalvalue': parse_column_options('decimal') }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -231,6 +234,7 @@ def test_required_field(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno'), 'text1': parse_column_options('habitat')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -253,6 +257,7 @@ def test_readonly_picklist(self) -> None: 'title': parse_column_options('title'), 'lastname': parse_column_options('lastname'), }, + overrideScope=None, static={'agenttype': 1}, toOne={}, toMany={} @@ -358,6 +363,7 @@ def test_agent_type(self) -> None: 'agenttype': parse_column_options('agenttype'), 'lastname': parse_column_options('lastname'), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -545,6 +551,7 @@ def test_wbcols_with_ignoreWhenBlank(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreWhenBlank", nullAllowed=True, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -570,6 +577,7 @@ def test_wbcols_with_ignoreWhenBlank_and_default(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreWhenBlank", nullAllowed=True, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -601,6 +609,7 @@ def test_wbcols_with_ignoreNever(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=True, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -625,6 +634,7 @@ def test_wbcols_with_ignoreAlways(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreAlways", nullAllowed=True, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -652,6 +662,7 @@ def test_wbcols_with_default(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=True, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -679,6 +690,7 @@ def test_wbcols_with_default_matching(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=True, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -709,6 +721,7 @@ def test_wbcols_with_default_and_null_disallowed(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=False, default="John"), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -737,6 +750,7 @@ def test_wbcols_with_default_blank(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=False, default=""), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -766,6 +780,7 @@ def test_wbcols_with_null_disallowed(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreNever", nullAllowed=False, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -790,6 +805,7 @@ def test_wbcols_with_null_disallowed_and_ignoreWhenBlank(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreWhenBlank", nullAllowed=False, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} @@ -818,6 +834,7 @@ def test_wbcols_with_null_disallowed_and_ignoreAlways(self) -> None: 'lastname': parse_column_options('lastname'), 'firstname': ColumnOptions(column='firstname', matchBehavior="ignoreAlways", nullAllowed=False, default=None), }, + overrideScope=None, static={}, toOne={}, toMany={} diff --git a/specifyweb/workbench/upload/tests/testschema.py b/specifyweb/workbench/upload/tests/testschema.py index 695a5803654..1a0aa8080e8 100644 --- a/specifyweb/workbench/upload/tests/testschema.py +++ b/specifyweb/workbench/upload/tests/testschema.py @@ -66,7 +66,7 @@ class OtherSchemaTests(unittest.TestCase): @given(name=infer, wbcols=infer) def test_validate_upload_table_to_json(self, name: str, wbcols: Dict[str, ColumnOptions]): - upload_table = UploadTable(name=name, wbcols=wbcols, static={}, toOne={}, toMany={}) + upload_table = UploadTable(name=name, wbcols=wbcols, overrideScope=None, static={}, toOne={}, toMany={}) validate(upload_table.unparse(), schema) @given(column_opts=from_schema(schema['definitions']['columnOptions'])) diff --git a/specifyweb/workbench/upload/tests/testuploading.py b/specifyweb/workbench/upload/tests/testuploading.py index f24dbf4fff4..9c1da62a454 100644 --- a/specifyweb/workbench/upload/tests/testuploading.py +++ b/specifyweb/workbench/upload/tests/testuploading.py @@ -328,11 +328,13 @@ def test_attachmentimageattribute(self) -> None: plan = UploadTable( name='Attachment', wbcols={'guid': parse_column_options('guid')}, + overrideScope=None, static={}, toMany={}, toOne={'attachmentimageattribute': UploadTable( name='Attachmentimageattribute', wbcols={'height': parse_column_options('height')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -354,11 +356,13 @@ def test_collectingtripattribute(self) -> None: plan = UploadTable( name='Collectingtrip', wbcols={'collectingtripname': parse_column_options('guid')}, + overrideScope=None, static={}, toMany={}, toOne={'collectingtripattribute': UploadTable( name='Collectingtripattribute', wbcols={'integer1': parse_column_options('integer')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -380,12 +384,14 @@ def test_preparationattribute(self) -> None: plan = UploadTable( name='Preparation', wbcols={'guid': parse_column_options('guid')}, + overrideScope=None, static={}, toMany={}, toOne={ 'preptype': UploadTable( name='Preptype', wbcols={'name': parse_column_options('preptype')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -393,6 +399,7 @@ def test_preparationattribute(self) -> None: 'preparationattribute': UploadTable( name='Preparationattribute', wbcols={'number1': parse_column_options('integer')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -400,6 +407,7 @@ def test_preparationattribute(self) -> None: 'collectionobject': UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -425,11 +433,13 @@ def test_collectionobjectattribute(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toMany={}, toOne={'collectionobjectattribute': UploadTable( name='Collectionobjectattribute', wbcols={'number1': parse_column_options('number')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -453,11 +463,13 @@ def test_collectingeventattribute(self) -> None: plan = UploadTable( name='Collectingevent', wbcols={'stationfieldnumber': parse_column_options('sfn')}, + overrideScope=None, static={}, toMany={}, toOne={'collectingeventattribute': UploadTable( name='Collectingeventattribute', wbcols={'number1': parse_column_options('number')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -534,6 +546,7 @@ def test_ambiguous_one_to_one_match(self) -> None: plan = UploadTable( name='Collectingevent', wbcols={'stationfieldnumber': parse_column_options('sfn')}, + overrideScope=None, static={}, toMany={}, toOne={'collectingeventattribute': UploadTable( @@ -556,11 +569,13 @@ def test_null_record_with_ambiguous_one_to_one(self) -> None: plan = UploadTable( name='Collectionobject', wbcols={'catalognumber': parse_column_options('catno')}, + overrideScope=None, static={}, toMany={}, toOne={'collectionobjectattribute': UploadTable( name='Collectionobjectattribute', wbcols={'number1': parse_column_options('number')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -667,6 +682,7 @@ def test_ordernumber(self) -> None: plan = UploadTable( name='Referencework', wbcols={'title': parse_column_options('title')}, + overrideScope=None, static={'referenceworktype': 0}, toOne={}, toMany={'authors': [ @@ -677,6 +693,7 @@ def test_ordernumber(self) -> None: toOne={'agent': UploadTable( name='Agent', wbcols={'lastname': parse_column_options('author1')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -688,6 +705,7 @@ def test_ordernumber(self) -> None: toOne={'agent': UploadTable( name='Agent', wbcols={'lastname': parse_column_options('author2')}, + overrideScope=None, static={}, toOne={}, toMany={} @@ -708,6 +726,7 @@ def test_no_override_ordernumber(self) -> None: plan = UploadTable( name='Referencework', wbcols={'title': parse_column_options('title')}, + overrideScope=None, static={'referenceworktype': 0}, toOne={}, toMany={'authors': [ @@ -718,6 +737,7 @@ def test_no_override_ordernumber(self) -> None: toOne={'agent': UploadTable( name='Agent', wbcols={'lastname': parse_column_options('author1')}, + overrideScope=None, static={}, toOne={}, toMany={} From 218d14d5c3595ab3fc1d73767595fed4a6183a9a Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 24 Mar 2023 16:09:17 -0500 Subject: [PATCH 09/31] Use proper imports for models.Collection --- specifyweb/workbench/upload/upload.py | 4 ++-- specifyweb/workbench/upload/upload_table.py | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 5dfc8b5e8a6..36a6a732f18 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -21,7 +21,7 @@ from .upload_result import Uploaded, UploadResult, ParseFailures, \ json_to_UploadResult from .uploadable import ScopedUploadable, Row, Disambiguation, Auditor -from ..models import Spdataset +from ..models import Spdataset, Collection Rows = Union[List[Row], csv.DictReader] Progress = Callable[[int, Optional[int]], None] @@ -192,7 +192,7 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: is_deferred = lambda uploadable: isinstance(uploadable, DeferredScopeUploadTable) - def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection: + def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> Collection: related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] related_column_name = related_uploadable.wbcols['name'][0] filter_value = rows[row_index][related_column_name] diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index d8654bd0921..fb224c74221 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -7,6 +7,7 @@ from specifyweb.businessrules.exceptions import BusinessRuleException from specifyweb.specify import models +from ..models import Collection from .column_options import ColumnOptions, ExtendedColumnOptions from .parsing import parse_many, ParseResult, ParseFailure from .tomany import ToManyRecord, ScopedToManyRecord, BoundToManyRecord @@ -74,7 +75,7 @@ class DeferredScopeUploadTable(NamedTuple): # (which follows the same logic as in UploadTable), or a function which has the parameter # signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection # (see apply_deferred_scopes in .upload.py) - overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], models.Collection]]]] + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Collection]]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] @@ -90,7 +91,7 @@ def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTa return apply_scoping_to_uploadtable(self, collection) else: return self - def add_colleciton_override(self, collection: Union[int, Callable[["DeferredScopeUploadTable", int], models.Collection]]) -> "DeferredScopeUploadTable": + def add_colleciton_override(self, collection: Union[int, Callable[["DeferredScopeUploadTable", int], Collection]]) -> "DeferredScopeUploadTable": ''' To modify the overrideScope after the DeferredScope UploadTable is created, use add_colleciton_override To properly apply scoping (see self.bind()), the should either be a collection's id, or a callable (function), which has paramaters that accept: this DeferredScope UploadTable, and an integer representing the current row_index. From 43015d8120a1ce7f0cc45d7e0853c3c295c79b4b Mon Sep 17 00:00:00 2001 From: melton-jason Date: Mon, 27 Mar 2023 13:32:39 -0500 Subject: [PATCH 10/31] Add remaining methods to DeferredScopeUploadTable from UploadTable --- specifyweb/workbench/upload/upload_table.py | 31 +++++++++++++++++++-- 1 file changed, 28 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index fb224c74221..645710e8d7f 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -7,7 +7,6 @@ from specifyweb.businessrules.exceptions import BusinessRuleException from specifyweb.specify import models -from ..models import Collection from .column_options import ColumnOptions, ExtendedColumnOptions from .parsing import parse_many, ParseResult, ParseFailure from .tomany import ToManyRecord, ScopedToManyRecord, BoundToManyRecord @@ -75,7 +74,7 @@ class DeferredScopeUploadTable(NamedTuple): # (which follows the same logic as in UploadTable), or a function which has the parameter # signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection # (see apply_deferred_scopes in .upload.py) - overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Collection]]]] + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Any]]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] @@ -91,7 +90,12 @@ def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTa return apply_scoping_to_uploadtable(self, collection) else: return self - def add_colleciton_override(self, collection: Union[int, Callable[["DeferredScopeUploadTable", int], Collection]]) -> "DeferredScopeUploadTable": + def get_cols(self) -> Set[str]: + return set(cd.column for cd in self.wbcols.values()) \ + | set(col for u in self.toOne.values() for col in u.get_cols()) \ + | set(col for rs in self.toMany.values() for r in rs for col in r.get_cols()) + + def add_colleciton_override(self, collection: Union[int, Callable[["DeferredScopeUploadTable", int], Any]]) -> "DeferredScopeUploadTable": ''' To modify the overrideScope after the DeferredScope UploadTable is created, use add_colleciton_override To properly apply scoping (see self.bind()), the should either be a collection's id, or a callable (function), which has paramaters that accept: this DeferredScope UploadTable, and an integer representing the current row_index. @@ -150,6 +154,27 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) + + def _to_json(self) -> Dict: + result = dict( + wbcols={k: v.to_json() for k,v in self.wbcols.items()}, + static=self.static + ) + result['toOne'] = { + key: uploadable.to_json() + for key, uploadable in self.toOne.items() + } + result['toMany'] = { + key: [to_many.to_json() for to_many in to_manys] + for key, to_manys in self.toMany.items() + } + return result + + def to_json(self) -> Dict: + return { 'uploadTable': self._to_json() } + + def unparse(self) -> Dict: + return { 'baseTableName': self.name, 'uploadable': self.to_json() } class ScopedUploadTable(NamedTuple): From 48d6367e44a241d4e15477b526af09817e461c6c Mon Sep 17 00:00:00 2001 From: melton-jason Date: Mon, 27 Mar 2023 14:17:26 -0500 Subject: [PATCH 11/31] Truly make overrideScope optional in UploadTable and UnuploadTable --- specifyweb/workbench/upload/scoping.py | 4 ---- specifyweb/workbench/upload/upload_table.py | 15 ++++++++------- 2 files changed, 8 insertions(+), 11 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 4df523e8413..f2964816276 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -104,10 +104,6 @@ def extend_columnoptions(colopts: ColumnOptions, collection, tablename: str, fie dateformat=get_date_format(), ) -def get_scoping_overrides(ut: UploadTable) -> Optional[int]: - if (ut.overrideScope is not None): - return ut.overrideScope - def apply_scoping_to_uploadtable(ut: UploadTable, collection) -> ScopedUploadTable: table = datamodel.get_table_strict(ut.name) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 645710e8d7f..ab59f17d806 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -21,12 +21,13 @@ class UploadTable(NamedTuple): name: str - overrideScope: Optional[Dict[Literal['collection'], Optional[int]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] toMany: Dict[str, List[ToManyRecord]] + overrideScope: Optional[Dict[Literal['collection'], Optional[int]]] = None + def apply_scoping(self, collection) -> "ScopedUploadTable": from .scoping import apply_scoping_to_uploadtable return apply_scoping_to_uploadtable(self, collection) @@ -69,12 +70,6 @@ class DeferredScopeUploadTable(NamedTuple): method is called. In which case, the rows of the dataset are known and the scoping can be deduced ''' name: str - - # In a DeferredScopeUploadTable, the overrideScope value can be either an integer - # (which follows the same logic as in UploadTable), or a function which has the parameter - # signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection - # (see apply_deferred_scopes in .upload.py) - overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Any]]]] wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] toOne: Dict[str, Uploadable] @@ -84,6 +79,12 @@ class DeferredScopeUploadTable(NamedTuple): relationship_name: str filter_field: str + # In a DeferredScopeUploadTable, the overrideScope value can be either an integer + # (which follows the same logic as in UploadTable), or a function which has the parameter + # signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection + # (see apply_deferred_scopes in .upload.py) + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Any]]]] = None + def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", "DeferredScopeUploadTable"]: if not defer: from .scoping import apply_scoping_to_uploadtable From 6f5210a2593965ab0f907d0dc1df9d886566619d Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 29 Mar 2023 15:32:19 -0500 Subject: [PATCH 12/31] Add test for inferring/parsing Deferred-Scope Upload Table --- .../workbench/upload/tests/testscoping.py | 92 ++++++++++++++++++- 1 file changed, 89 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/tests/testscoping.py b/specifyweb/workbench/upload/tests/testscoping.py index 07b2cf5a56f..64b9da544a2 100644 --- a/specifyweb/workbench/upload/tests/testscoping.py +++ b/specifyweb/workbench/upload/tests/testscoping.py @@ -1,8 +1,8 @@ -from ..upload_plan_schema import schema, parse_plan -from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable +from ..upload_plan_schema import schema, parse_plan, parse_plan_with_basetable +from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable, DeferredScopeUploadTable, ColumnOptions -from .base import UploadTestsBase +from .base import UploadTestsBase, get_table from . import example_plan @@ -47,3 +47,89 @@ def test_embedded_paleocontext_in_collectionobject(self) -> None: ).apply_scoping(self.collection) self.assertIsInstance(plan.toOne['paleocontext'], ScopedOneToOneTable) + + def collection_rel_type_being_deferred(self) -> None: + + unparsed_upload_plan = { + "baseTableName": "collectionrelationship", + "uploadable": { + "uploadTable": { + "wbcols": {}, + "static": {}, + "toOne": { + "leftside": { + "uploadTable": { + "wbcols": { + "catalognumber": "Cat #" + }, + "static": {}, + "toOne": {}, + "toMany": {} + } + }, + "rightside": { + "uploadTable": { + "wbcols": { + "catalognumber": "Cat # (2)" + }, + "static": {}, + "toOne": {}, + "toMany": {} + } + }, + "collectionreltype": { + "uploadTable": { + "wbcols": { + "name": "Collection Rel Type" + }, + "static": {}, + "toOne": {}, + "toMany": {} + } + } + }, + "toMany": {} + } + } + } + table, parsed_plan = parse_plan_with_basetable(self.collection, unparsed_upload_plan) + + expected_plan = UploadTable( + name='Collectionrelationship', + wbcols={}, + static={}, + toOne={ + 'leftside': UploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat #', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + overrideScope=None + ), + + 'rightside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat # (2)', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='rightsidecollection', + filter_field='name', + overrideScope=None + ), + + 'collectionreltype': UploadTable( + name='Collectionreltype', + wbcols={'name': ColumnOptions(column='Collection Rel Type', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + overrideScope=None + ) + }, + toMany={}, + overrideScope=None) + + self.assertEqual(parsed_plan, expected_plan) From 21418bcc4911205100ac95c8b518602708d873b0 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 29 Mar 2023 15:35:07 -0500 Subject: [PATCH 13/31] Remove direct 'toOne' attribute access --- specifyweb/workbench/upload/upload.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 36a6a732f18..8b14c5cbacb 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -190,7 +190,6 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab return base_table, plan.apply_scoping(collection) def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: - is_deferred = lambda uploadable: isinstance(uploadable, DeferredScopeUploadTable) def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> Collection: related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] @@ -206,8 +205,8 @@ def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, if hasattr(upload_plan, 'toOne'): for key, uploadable in upload_plan.toOne.items(): _uploadable = uploadable - if _uploadable.toOne != {}: _uploadable = apply_deferred_scopes(_uploadable, rows) - if is_deferred(_uploadable): + if hasattr(_uploadable, 'toOne'): _uploadable = apply_deferred_scopes(_uploadable, rows) + if isinstance(_uploadable, DeferredScopeUploadTable): _uploadable = _uploadable.add_colleciton_override(collection_override_function) upload_plan.toOne[key] = _uploadable From 076cb1d4f62ce339a381d454b0b81494f6235a51 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 29 Mar 2023 16:18:55 -0500 Subject: [PATCH 14/31] Make rightSide unique in CollectionRelationship to Collectionreltype --- specifyweb/businessrules/uniqueness_rules.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/specifyweb/businessrules/uniqueness_rules.py b/specifyweb/businessrules/uniqueness_rules.py index 4479f5d846d..239118524f3 100644 --- a/specifyweb/businessrules/uniqueness_rules.py +++ b/specifyweb/businessrules/uniqueness_rules.py @@ -71,6 +71,9 @@ def check_unique(instance): 'Collectionobject': { 'catalognumber': ['collection'], }, + 'Collectionrelationship' : { + 'rightside' : ['collectionreltype'] + }, 'Collector': { 'agent': ['collectingevent'], }, From 56dc2a5f07ad8cfeb7e83207ecea0b19d2800897 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 31 Mar 2023 08:29:10 -0500 Subject: [PATCH 15/31] Always use specified leftsidecollection from collectionreltype when uploading collectionrealtionship --- specifyweb/workbench/upload/scoping.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index f2964816276..55abac5a2a3 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -34,7 +34,10 @@ See .upload_plan_schema.py for how this is used """ -DEFERRED_SCOPING: Dict[Tuple[str, str], Tuple[str, str, str]] = {("Collectionrelationship", "rightside"): ('collectionreltype', 'name', 'rightsidecollection')} +DEFERRED_SCOPING: Dict[Tuple[str, str], Tuple[str, str, str]] = { + ("Collectionrelationship", "rightside"): ('collectionreltype', 'name', 'rightsidecollection'), + ("Collectionrelationship", "leftside"): ('collectionreltype', 'name', 'leftsidecollection'), + } def scoping_relationships(collection, table: Table) -> Dict[str, int]: extra_static: Dict[str, int] = {} From 623718cab43045bcdff23a2c6e1c22b1e13da3fd Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 31 Mar 2023 12:38:36 -0500 Subject: [PATCH 16/31] Finish collection relationship workbench tests --- .../workbench/upload/tests/testscoping.py | 190 +++++++++++++----- 1 file changed, 139 insertions(+), 51 deletions(-) diff --git a/specifyweb/workbench/upload/tests/testscoping.py b/specifyweb/workbench/upload/tests/testscoping.py index 64b9da544a2..48d520b163a 100644 --- a/specifyweb/workbench/upload/tests/testscoping.py +++ b/specifyweb/workbench/upload/tests/testscoping.py @@ -1,56 +1,30 @@ +from ..upload_plan_schema import schema, parse_plan +from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable, DeferredScopeUploadTable, ColumnOptions, ExtendedColumnOptions +from ..upload import do_upload -from ..upload_plan_schema import schema, parse_plan, parse_plan_with_basetable -from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable, DeferredScopeUploadTable, ColumnOptions - +from specifyweb.specify.models import Splocalecontaineritem, Splocalecontainer, Collectionobject, Collectionrelationship from .base import UploadTestsBase, get_table from . import example_plan class ScopingTests(UploadTestsBase): - def test_embedded_collectingevent(self) -> None: - self.collection.isembeddedcollectingevent = True - self.collection.save() - - plan = parse_plan(self.collection, example_plan.json) - - assert isinstance(plan, UploadTable) - ce_rel = plan.toOne['collectingevent'] - - self.assertNotIsInstance(ce_rel, OneToOneTable) - - scoped = plan.apply_scoping(self.collection) - - assert isinstance(scoped, ScopedUploadTable) - scoped_ce_rel = scoped.toOne['collectingevent'] - - self.assertIsInstance(scoped_ce_rel, ScopedOneToOneTable) - - - def test_embedded_paleocontext_in_collectionobject(self) -> None: - self.collection.discipline.ispaleocontextembedded = True - self.collection.discipline.paleocontextchildtable = 'collectionobject' - self.collection.save() - - plan = UploadTable( - name='Collectionobject', - toOne={'paleocontext': UploadTable( - name='Paleocontext', - wbcols={}, - toOne={}, - toMany={}, - static={}, - )}, - wbcols={}, - static={}, - toMany={}, - ).apply_scoping(self.collection) + def setUp(self) -> None: + self.rel_type_name = "ToRightSide" - self.assertIsInstance(plan.toOne['paleocontext'], ScopedOneToOneTable) + self.right_side_collection = get_table('Collection').objects.create( + catalognumformatname='test', + collectionname='RightSideTest', + isembeddedcollectingevent=False, + discipline=self.discipline) - def collection_rel_type_being_deferred(self) -> None: + get_table('Collectionreltype').objects.create( + name = self.rel_type_name, + leftsidecollection = self.collection, + rightsidecollection = self.right_side_collection, + ) - unparsed_upload_plan = { + self.collection_rel_plan = { "baseTableName": "collectionrelationship", "uploadable": { "uploadTable": { @@ -92,22 +66,70 @@ def collection_rel_type_being_deferred(self) -> None: } } } - table, parsed_plan = parse_plan_with_basetable(self.collection, unparsed_upload_plan) + + + return super().setUp() + + def test_embedded_collectingevent(self) -> None: + self.collection.isembeddedcollectingevent = True + self.collection.save() + + plan = parse_plan(self.collection, example_plan.json) + + assert isinstance(plan, UploadTable) + ce_rel = plan.toOne['collectingevent'] + + self.assertNotIsInstance(ce_rel, OneToOneTable) + + scoped = plan.apply_scoping(self.collection) + + assert isinstance(scoped, ScopedUploadTable) + scoped_ce_rel = scoped.toOne['collectingevent'] + + self.assertIsInstance(scoped_ce_rel, ScopedOneToOneTable) + + + def test_embedded_paleocontext_in_collectionobject(self) -> None: + self.collection.discipline.ispaleocontextembedded = True + self.collection.discipline.paleocontextchildtable = 'collectionobject' + self.collection.save() + + plan = UploadTable( + name='Collectionobject', + toOne={'paleocontext': UploadTable( + name='Paleocontext', + wbcols={}, + toOne={}, + toMany={}, + static={}, + )}, + wbcols={}, + static={}, + toMany={}, + ).apply_scoping(self.collection) + + self.assertIsInstance(plan.toOne['paleocontext'], ScopedOneToOneTable) + + def collection_rel_type_being_deferred(self) -> None: + + parsed_plan = parse_plan(self.collection, self.collection_rel_plan) expected_plan = UploadTable( name='Collectionrelationship', wbcols={}, static={}, toOne={ - 'leftside': UploadTable( + 'leftside': DeferredScopeUploadTable( name='Collectionobject', - wbcols={'catalognumber': ColumnOptions(column='Cat #', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, - static={}, - toOne={}, - toMany={}, + wbcols={'catalognumber': ColumnOptions(column='Cat #', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='leftsidecollection', + filter_field='name', overrideScope=None ), - 'rightside': DeferredScopeUploadTable( name='Collectionobject', wbcols={'catalognumber': ColumnOptions(column='Cat # (2)', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, @@ -119,7 +141,6 @@ def collection_rel_type_being_deferred(self) -> None: filter_field='name', overrideScope=None ), - 'collectionreltype': UploadTable( name='Collectionreltype', wbcols={'name': ColumnOptions(column='Collection Rel Type', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, @@ -133,3 +154,70 @@ def collection_rel_type_being_deferred(self) -> None: overrideScope=None) self.assertEqual(parsed_plan, expected_plan) + + def deferred_scope_table_ignored_when_scoping_applied(self): + scoped_upload_plan = parse_plan(self.collection_rel_plan).apply_scoping(self.collection) + + expected_scoping = ScopedUploadTable( + name='Collectionrelationship', + wbcols={}, + static={}, + toOne={ + 'leftside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat #', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='leftsidecollection', + filter_field='name', + overrideScope=None), + 'rightside': DeferredScopeUploadTable( + name='Collectionobject', + wbcols={'catalognumber': ColumnOptions(column='Cat # (2)', matchBehavior='ignoreNever', nullAllowed=True, default=None)}, + static={}, + toOne={}, + toMany={}, + related_key='collectionreltype', + relationship_name='rightsidecollection', + filter_field='name', + overrideScope=None), + 'collectionreltype': ScopedUploadTable( + name='Collectionreltype', + wbcols={'name': ExtendedColumnOptions( + column='Collection Rel Type', + matchBehavior='ignoreNever', + nullAllowed=True, + default=None, + uiformatter=None, + schemaitem= Splocalecontaineritem.objects.get(name='name', container=Splocalecontainer.objects.get(name='collectionreltype', discipline_id=self.discipline.id)), + picklist=None, + dateformat='%m/%d/%Y')}, + static={}, + toOne={}, + toMany={}, + scopingAttrs={}, + disambiguation=None)}, + + toMany={}, + scopingAttrs={}, + disambiguation=None) + + self.assertEqual(scoped_upload_plan, expected_scoping) + + def collection_rel_uploaded_in_correct_collection(self): + scoped_plan = parse_plan(self.collection_rel_plan).apply_scoping(self.collection) + rows = [ + {'Collection Rel Type': self.rel_type_name, 'Cat # (2)': '999', 'Cat #': '23'}, + {'Collection Rel Type': self.rel_type_name, 'Cat # (2)': '888', 'Cat #': '32'} + ] + do_upload(self.collection, rows, scoped_plan, self.agent.id) + left_side_cat_nums = [n.zfill(9) for n in '32 23'.split()] + right_side_cat_nums = [n.zfill(9) for n in '999 888'.split()] + + left_side_query = Collectionobject.objects.filter(collection_id=self.collection.id, catalognumber__in=left_side_cat_nums) + right_side_query = Collectionobject.objects.filter(collection_id=self.right_side_collection.id, catalognumber__in=right_side_cat_nums) + + self.assertEqual(left_side_query.count(), 2) + self.assertEqual(right_side_query.count(), 2) From 827ff61c20fb5fc2de489c8324e901324f9473ff Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 5 Apr 2023 14:54:22 -0500 Subject: [PATCH 17/31] Fix backend test failing due to recursive type More specifically, our version of mypy has a known issue regarding processing the type hints for subclasses of NamedTuples which reference themselves. i.e., using the "DeferredScopeUploadTable" type hint in the DeferredScopeUploadTable class For more details, see https://github.com/python/mypy/issues/8695 --- specifyweb/workbench/upload/upload_table.py | 31 +++++++++++++++------ 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index ab59f17d806..dcd63aab53e 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -1,7 +1,7 @@ import logging from functools import reduce -from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal +from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal, get_type_hints from django.db import transaction, IntegrityError @@ -79,13 +79,23 @@ class DeferredScopeUploadTable(NamedTuple): relationship_name: str filter_field: str - # In a DeferredScopeUploadTable, the overrideScope value can be either an integer - # (which follows the same logic as in UploadTable), or a function which has the parameter - # signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection - # (see apply_deferred_scopes in .upload.py) - overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Any]]]] = None + """ In a DeferredScopeUploadTable, the overrideScope value can be either an integer + (which follows the same logic as in UploadTable), or a function which has the parameter + signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection + (see apply_deferred_scopes in .upload.py) - def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", "DeferredScopeUploadTable"]: + overrideScope should be of type + Optional[Dict[Literal["collection"], Union[int, Callable[["DeferredScopeUploadTable", int], Any]]]] + + But recursively using the type within the class definition of a NamedTuple is not supported in our version + of mypy + See https://github.com/python/mypy/issues/8695 + """ + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[[NamedTuple, int], Any]]]] = None + + + # Typehint for return type should be: Union["ScopedUploadTable", "DeferredScopeUploadTable"] + def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", NamedTuple]: if not defer: from .scoping import apply_scoping_to_uploadtable return apply_scoping_to_uploadtable(self, collection) @@ -96,7 +106,12 @@ def get_cols(self) -> Set[str]: | set(col for u in self.toOne.values() for col in u.get_cols()) \ | set(col for rs in self.toMany.values() for r in rs for col in r.get_cols()) - def add_colleciton_override(self, collection: Union[int, Callable[["DeferredScopeUploadTable", int], Any]]) -> "DeferredScopeUploadTable": + + """ + The Typehint for parameter collection should be: Union[int, Callable[["DeferredScopeUploadTable", int], Any]] + The Typehint for return type should be: "DeferredScopeUploadTable" + """ + def add_colleciton_override(self, collection: Union[int, Callable[[NamedTuple, int], Any]]) -> NamedTuple: ''' To modify the overrideScope after the DeferredScope UploadTable is created, use add_colleciton_override To properly apply scoping (see self.bind()), the should either be a collection's id, or a callable (function), which has paramaters that accept: this DeferredScope UploadTable, and an integer representing the current row_index. From 8d89155b4b8dbc287a73421248251b63fd64e02f Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 5 Apr 2023 20:05:56 +0000 Subject: [PATCH 18/31] Lint code with ESLint and Prettier Triggered by 827ff61c20fb5fc2de489c8324e901324f9473ff on branch refs/heads/issue-3089 --- .../frontend/js_src/lib/components/AppResources/Aside.tsx | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/AppResources/Aside.tsx b/specifyweb/frontend/js_src/lib/components/AppResources/Aside.tsx index 2f37ecf836e..6e25b02f704 100644 --- a/specifyweb/frontend/js_src/lib/components/AppResources/Aside.tsx +++ b/specifyweb/frontend/js_src/lib/components/AppResources/Aside.tsx @@ -214,7 +214,7 @@ function TreeItem({ > handleFold( @@ -226,7 +226,9 @@ function TreeItem({ > {count}, + wrap: (count) => ( + {count} + ), }} string={commonText.jsxCountLine({ resource: label, From a933fb71e90d90ab8234a31e254054b465242462 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 5 Apr 2023 15:07:48 -0500 Subject: [PATCH 19/31] Use type Any instead of NamedTuple in DeferredcopeUploadTable --- specifyweb/workbench/upload/upload_table.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index dcd63aab53e..f8fdd121721 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -1,7 +1,7 @@ import logging from functools import reduce -from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal, get_type_hints +from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal from django.db import transaction, IntegrityError @@ -91,11 +91,11 @@ class DeferredScopeUploadTable(NamedTuple): of mypy See https://github.com/python/mypy/issues/8695 """ - overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[[NamedTuple, int], Any]]]] = None + overrideScope: Optional[Dict[Literal["collection"], Union[int, Callable[[Any, int], Any]]]] = None # Typehint for return type should be: Union["ScopedUploadTable", "DeferredScopeUploadTable"] - def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", NamedTuple]: + def apply_scoping(self, collection, defer: bool = True) -> Union["ScopedUploadTable", Any]: if not defer: from .scoping import apply_scoping_to_uploadtable return apply_scoping_to_uploadtable(self, collection) @@ -111,7 +111,7 @@ def get_cols(self) -> Set[str]: The Typehint for parameter collection should be: Union[int, Callable[["DeferredScopeUploadTable", int], Any]] The Typehint for return type should be: "DeferredScopeUploadTable" """ - def add_colleciton_override(self, collection: Union[int, Callable[[NamedTuple, int], Any]]) -> NamedTuple: + def add_colleciton_override(self, collection: Union[int, Callable[[Any, int], Any]]) -> Any: ''' To modify the overrideScope after the DeferredScope UploadTable is created, use add_colleciton_override To properly apply scoping (see self.bind()), the should either be a collection's id, or a callable (function), which has paramaters that accept: this DeferredScope UploadTable, and an integer representing the current row_index. From 9d559c09341213c44faaf62c5178f68fd25ce723 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 5 Apr 2023 15:32:21 -0500 Subject: [PATCH 20/31] Fix most of failing backend tests --- specifyweb/workbench/upload/scoping.py | 6 +++--- specifyweb/workbench/upload/tests/testscoping.py | 8 ++++---- specifyweb/workbench/upload/upload.py | 8 ++++---- specifyweb/workbench/upload/upload_table.py | 6 ++++-- 4 files changed, 15 insertions(+), 13 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 55abac5a2a3..98318f7c1f1 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -1,4 +1,4 @@ -from typing import Dict, Any, Optional, Tuple, Callable +from typing import Dict, Any, Optional, Tuple, Callable, Union from specifyweb.specify.datamodel import datamodel, Table, Relationship from specifyweb.specify.load_datamodel import DoesNotExistError @@ -7,7 +7,7 @@ from specifyweb.stored_queries.format import get_date_format from .uploadable import Uploadable, ScopedUploadable -from .upload_table import UploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable +from .upload_table import UploadTable, DeferredScopeUploadTable, ScopedUploadTable, OneToOneTable, ScopedOneToOneTable from .tomany import ToManyRecord, ScopedToManyRecord from .treerecord import TreeRecord, ScopedTreeRecord from .column_options import ColumnOptions, ExtendedColumnOptions @@ -107,7 +107,7 @@ def extend_columnoptions(colopts: ColumnOptions, collection, tablename: str, fie dateformat=get_date_format(), ) -def apply_scoping_to_uploadtable(ut: UploadTable, collection) -> ScopedUploadTable: +def apply_scoping_to_uploadtable(ut: Union[UploadTable, DeferredScopeUploadTable], collection) -> ScopedUploadTable: table = datamodel.get_table_strict(ut.name) adjust_to_ones = to_one_adjustments(collection, table) diff --git a/specifyweb/workbench/upload/tests/testscoping.py b/specifyweb/workbench/upload/tests/testscoping.py index 48d520b163a..b4fdf156c6b 100644 --- a/specifyweb/workbench/upload/tests/testscoping.py +++ b/specifyweb/workbench/upload/tests/testscoping.py @@ -2,7 +2,7 @@ from ..upload_table import UploadTable, OneToOneTable, ScopedUploadTable, ScopedOneToOneTable, DeferredScopeUploadTable, ColumnOptions, ExtendedColumnOptions from ..upload import do_upload -from specifyweb.specify.models import Splocalecontaineritem, Splocalecontainer, Collectionobject, Collectionrelationship +from specifyweb.specify import models from .base import UploadTestsBase, get_table from . import example_plan @@ -191,7 +191,7 @@ def deferred_scope_table_ignored_when_scoping_applied(self): nullAllowed=True, default=None, uiformatter=None, - schemaitem= Splocalecontaineritem.objects.get(name='name', container=Splocalecontainer.objects.get(name='collectionreltype', discipline_id=self.discipline.id)), + schemaitem= models.Splocalecontaineritem.objects.get(name='name', container=models.Splocalecontainer.objects.get(name='collectionreltype', discipline_id=self.discipline.id)), picklist=None, dateformat='%m/%d/%Y')}, static={}, @@ -216,8 +216,8 @@ def collection_rel_uploaded_in_correct_collection(self): left_side_cat_nums = [n.zfill(9) for n in '32 23'.split()] right_side_cat_nums = [n.zfill(9) for n in '999 888'.split()] - left_side_query = Collectionobject.objects.filter(collection_id=self.collection.id, catalognumber__in=left_side_cat_nums) - right_side_query = Collectionobject.objects.filter(collection_id=self.right_side_collection.id, catalognumber__in=right_side_cat_nums) + left_side_query = models.Collectionobject.objects.filter(collection_id=self.collection.id, catalognumber__in=left_side_cat_nums) + right_side_query = models.Collectionobject.objects.filter(collection_id=self.right_side_collection.id, catalognumber__in=right_side_cat_nums) self.assertEqual(left_side_query.count(), 2) self.assertEqual(right_side_query.count(), 2) diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 8b14c5cbacb..e0ead7651bf 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -15,13 +15,13 @@ from specifyweb.specify.auditlog import auditlog from specifyweb.specify.datamodel import Table from specifyweb.specify.tree_extras import renumber_tree, reset_fullnames -from specifyweb.workbench.upload.upload_table import DeferredScopeUploadTable +from specifyweb.workbench.upload.upload_table import DeferredScopeUploadTable, ScopedUploadTable from . import disambiguation from .upload_plan_schema import schema, parse_plan_with_basetable from .upload_result import Uploaded, UploadResult, ParseFailures, \ json_to_UploadResult from .uploadable import ScopedUploadable, Row, Disambiguation, Auditor -from ..models import Spdataset, Collection +from specifyweb.specify.models import Spdataset Rows = Union[List[Row], csv.DictReader] Progress = Callable[[int, Optional[int]], None] @@ -189,9 +189,9 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab base_table, plan = parse_plan_with_basetable(collection, plan) return base_table, plan.apply_scoping(collection) -def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: +def apply_deferred_scopes(upload_plan: ScopedUploadTable, rows: Rows) -> ScopedUploadTable: - def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> Collection: + def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection: related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] related_column_name = related_uploadable.wbcols['name'][0] filter_value = rows[row_index][related_column_name] diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index f8fdd121721..cc1b910aa43 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -72,13 +72,15 @@ class DeferredScopeUploadTable(NamedTuple): name: str wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] - toOne: Dict[str, Uploadable] - toMany: Dict[str, List[ToManyRecord]] + toOne: Dict[str, Union[Uploadable, ScopedUploadable]] + toMany: Dict[str, List[Union[ToManyRecord, ScopedToManyRecord]]] related_key: str relationship_name: str filter_field: str + disambiguation: Disambiguation = None + """ In a DeferredScopeUploadTable, the overrideScope value can be either an integer (which follows the same logic as in UploadTable), or a function which has the parameter signature: (deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection From 5fb29b6b46682cdabf50d5019099ee5449a53db4 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Wed, 5 Apr 2023 17:03:29 -0500 Subject: [PATCH 21/31] Use correct import for Spdataset --- specifyweb/workbench/upload/upload.py | 2 +- specifyweb/workbench/upload/upload_table.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index e0ead7651bf..34d7cc3e4e9 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -21,7 +21,7 @@ from .upload_result import Uploaded, UploadResult, ParseFailures, \ json_to_UploadResult from .uploadable import ScopedUploadable, Row, Disambiguation, Auditor -from specifyweb.specify.models import Spdataset +from ..models import Spdataset Rows = Union[List[Row], csv.DictReader] Progress = Callable[[int, Optional[int]], None] diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index cc1b910aa43..ff293fd0087 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -72,7 +72,7 @@ class DeferredScopeUploadTable(NamedTuple): name: str wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] - toOne: Dict[str, Union[Uploadable, ScopedUploadable]] + toOne: Dict[str, Union[UploadTable, "ScopedUploadTable"]] toMany: Dict[str, List[Union[ToManyRecord, ScopedToManyRecord]]] related_key: str @@ -153,7 +153,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud Otherwise, if a funciton is provided (see apply_deferred_scopes in .upload.py), then call the function with the row and row_index to get the needed collection ''' - if 'collection' in self.overrideScope.keys(): + if 'collection' in self.overrideScope.keys(): # type: ignore if isinstance(self.overrideScope['collection'], int): collection_id = self.overrideScope['collection'] collection = models.Collection.objects.get(id=collection_id) From 04b1ea65d5f5252e1cfef3b240015b6e914848a6 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 7 Apr 2023 14:00:34 -0500 Subject: [PATCH 22/31] Fix more failing type check tests --- specifyweb/workbench/upload/upload.py | 19 +++++++++++-------- specifyweb/workbench/upload/upload_table.py | 8 ++++---- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 34d7cc3e4e9..77b2dac7f04 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -189,18 +189,21 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab base_table, plan = parse_plan_with_basetable(collection, plan) return base_table, plan.apply_scoping(collection) -def apply_deferred_scopes(upload_plan: ScopedUploadTable, rows: Rows) -> ScopedUploadTable: +def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: - def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int) -> models.Collection: - related_uploadable = upload_plan.toOne[deferred_upload_plan.related_key] + def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int): # -> models.Collection + related_uploadable: Union[ScopedUploadTable, DeferredScopeUploadTable] = upload_plan.toOne[deferred_upload_plan.related_key] related_column_name = related_uploadable.wbcols['name'][0] - filter_value = rows[row_index][related_column_name] + filter_value = rows[row_index][related_column_name] # type: ignore filter_search = {deferred_upload_plan.filter_field : filter_value} - related = getattr(models, datamodel.get_table(deferred_upload_plan.related_key).django_name).objects.get(**filter_search) - collection_id = getattr(related, deferred_upload_plan.relationship_name).id - collection = models.Collection.objects.get(id=collection_id) - return collection + + related_table = datamodel.get_table(deferred_upload_plan.related_key) + if related_table is not None: + related = getattr(models, related_table.django_name).objects.get(**filter_search) + collection_id = getattr(related, deferred_upload_plan.relationship_name).id + collection = models.Collection.objects.get(id=collection_id) + return collection if hasattr(upload_plan, 'toOne'): for key, uploadable in upload_plan.toOne.items(): diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index ff293fd0087..2104984f3bb 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -72,8 +72,8 @@ class DeferredScopeUploadTable(NamedTuple): name: str wbcols: Dict[str, ColumnOptions] static: Dict[str, Any] - toOne: Dict[str, Union[UploadTable, "ScopedUploadTable"]] - toMany: Dict[str, List[Union[ToManyRecord, ScopedToManyRecord]]] + toOne: Dict[str, Uploadable] + toMany: Dict[str, List[ToManyRecord]] related_key: str relationship_name: str @@ -153,7 +153,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud Otherwise, if a funciton is provided (see apply_deferred_scopes in .upload.py), then call the function with the row and row_index to get the needed collection ''' - if 'collection' in self.overrideScope.keys(): # type: ignore + if self.overrideScope is not None and'collection' in self.overrideScope.keys(): if isinstance(self.overrideScope['collection'], int): collection_id = self.overrideScope['collection'] collection = models.Collection.objects.get(id=collection_id) @@ -168,7 +168,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new # ScopedUploadTable - scoped_disambiguated = scoped.disambiguate(self.da) if hasattr(self, "disambiguation") else scoped + scoped_disambiguated = scoped.disambiguate(self.disambiguation) if self.disambiguate is not None else scoped # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) From b4c3ea38106424ab1e442d583220eac5291c7fcf Mon Sep 17 00:00:00 2001 From: Jason Melton <64045831+melton-jason@users.noreply.github.com> Date: Fri, 7 Apr 2023 19:18:50 +0000 Subject: [PATCH 23/31] Lint code with ESLint and Prettier Triggered by 690376b4803434a44d0514bc5440407414a6b09e on branch refs/heads/issue-3089 --- specifyweb/frontend/js_src/lib/components/Header/index.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/frontend/js_src/lib/components/Header/index.tsx b/specifyweb/frontend/js_src/lib/components/Header/index.tsx index 491700b5a6d..f81c18404c9 100644 --- a/specifyweb/frontend/js_src/lib/components/Header/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/Header/index.tsx @@ -19,12 +19,12 @@ import { MenuContext } from '../Core/Main'; import { schema } from '../DataModel/schema'; import { userInformation } from '../InitialContext/userInformation'; import { titleDelay, titlePosition } from '../Molecules/Tooltips'; +import { userPreferences } from '../Preferences/userPreferences'; import { ActiveLink } from '../Router/ActiveLink'; import type { MenuItemName } from './menuItemDefinitions'; import { useUserTools } from './menuItemProcessing'; import { Notifications } from './Notifications'; import { UserTools } from './UserTools'; -import { userPreferences } from '../Preferences/userPreferences'; const collapseThreshold = 900; From 7456aef65ef2ba065ca7abfcf1b2df5b54223a4a Mon Sep 17 00:00:00 2001 From: melton-jason Date: Fri, 7 Apr 2023 16:24:52 -0500 Subject: [PATCH 24/31] Fix more failing mypy type tests --- specifyweb/workbench/upload/scoping.py | 2 +- specifyweb/workbench/upload/upload.py | 16 +++++++++++----- specifyweb/workbench/upload/upload_table.py | 14 ++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/specifyweb/workbench/upload/scoping.py b/specifyweb/workbench/upload/scoping.py index 98318f7c1f1..bee5dfe09e3 100644 --- a/specifyweb/workbench/upload/scoping.py +++ b/specifyweb/workbench/upload/scoping.py @@ -113,7 +113,7 @@ def apply_scoping_to_uploadtable(ut: Union[UploadTable, DeferredScopeUploadTable adjust_to_ones = to_one_adjustments(collection, table) if ut.overrideScope is not None and isinstance(ut.overrideScope['collection'], int): - collection = models.Collection.objects.filter(id=ut.overrideScope['collection']).get() + collection = getattr(models, "Collection").objects.filter(id=ut.overrideScope['collection']).get() return ScopedUploadTable( diff --git a/specifyweb/workbench/upload/upload.py b/specifyweb/workbench/upload/upload.py index 77b2dac7f04..d16b63065b6 100644 --- a/specifyweb/workbench/upload/upload.py +++ b/specifyweb/workbench/upload/upload.py @@ -192,7 +192,8 @@ def get_ds_upload_plan(collection, ds: Spdataset) -> Tuple[Table, ScopedUploadab def apply_deferred_scopes(upload_plan: ScopedUploadable, rows: Rows) -> ScopedUploadable: def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, row_index: int): # -> models.Collection - related_uploadable: Union[ScopedUploadTable, DeferredScopeUploadTable] = upload_plan.toOne[deferred_upload_plan.related_key] + # to call this function, we always know upload_plan is either a DeferredScopeUploadTable or ScopedUploadTable + related_uploadable: Union[ScopedUploadTable, DeferredScopeUploadTable] = upload_plan.toOne[deferred_upload_plan.related_key] # type: ignore related_column_name = related_uploadable.wbcols['name'][0] filter_value = rows[row_index][related_column_name] # type: ignore @@ -202,16 +203,21 @@ def collection_override_function(deferred_upload_plan: DeferredScopeUploadTable, if related_table is not None: related = getattr(models, related_table.django_name).objects.get(**filter_search) collection_id = getattr(related, deferred_upload_plan.relationship_name).id - collection = models.Collection.objects.get(id=collection_id) + collection = getattr(models, "Collection").objects.get(id=collection_id) return collection if hasattr(upload_plan, 'toOne'): - for key, uploadable in upload_plan.toOne.items(): + # Without type ignores, MyPy throws the following error: "ScopedUploadable" has no attribute "toOne" + # MyPy expects upload_plan to be of type ScopedUploadable (from the paramater type) + # but within this if-statement we know that upload_plan is always an UploadTable + # (or more specifically, one if its derivatives: DeferredScopeUploadTable or ScopedUploadTable) + + for key, uploadable in upload_plan.toOne.items(): # type: ignore _uploadable = uploadable if hasattr(_uploadable, 'toOne'): _uploadable = apply_deferred_scopes(_uploadable, rows) if isinstance(_uploadable, DeferredScopeUploadTable): _uploadable = _uploadable.add_colleciton_override(collection_override_function) - upload_plan.toOne[key] = _uploadable + upload_plan.toOne[key] = _uploadable # type: ignore return upload_plan @@ -253,7 +259,7 @@ def do_upload( if no_commit: raise Rollback("no_commit option") else: - fixup_trees(upload_plan, results) + fixup_trees(deffered_upload_plan, results) return results diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 2104984f3bb..c0caa573b3a 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -135,11 +135,14 @@ def disambiguate(self, da: Disambiguation): then disambiguate the new Scoped UploadTable using the stored Disambiguation ''' return self._replace(disambiguation = da) - + def get_treedefs(self) -> Set: + """ This method is needed because a ScopedUploadTable may call this when calling its own get_treedefs() + This returns an empty set unless the toOne or toMany Uploadable is a TreeRecord + """ return ( - set(td for toOne in self.toOne.values() for td in toOne.get_treedefs()) | - set(td for toMany in self.toMany.values() for tmr in toMany for td in tmr.get_treedefs()) + set(td for toOne in self.toOne.values() for td in toOne.get_treedefs()) | # type: ignore + set(td for toMany in self.toMany.values() for tmr in toMany for td in tmr.get_treedefs()) # type: ignore ) def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Auditor, cache: Optional[Dict]=None, row_index: Optional[int] = None @@ -159,7 +162,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud collection = models.Collection.objects.get(id=collection_id) scoped = self.apply_scoping(collection, defer=False) elif isinstance(self.overrideScope['collection'], Callable): - collection = self.overrideScope['collection'](self, row_index) + collection = self.overrideScope['collection'](self, row_index) if row_index is not None else default_collection scoped = self.apply_scoping(collection, defer=False) # If the collection/scope should not be overriden, defer to the default behavior and assume @@ -168,8 +171,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new # ScopedUploadTable - scoped_disambiguated = scoped.disambiguate(self.disambiguation) if self.disambiguate is not None else scoped - + scoped_disambiguated = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) From ed1eabb4fdba4aa15b98fb877156f69376d19567 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Sun, 9 Apr 2023 11:40:54 -0500 Subject: [PATCH 25/31] Fix remaining type errors --- specifyweb/workbench/upload/upload_table.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index c0caa573b3a..841a8c622b7 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -159,9 +159,9 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud if self.overrideScope is not None and'collection' in self.overrideScope.keys(): if isinstance(self.overrideScope['collection'], int): collection_id = self.overrideScope['collection'] - collection = models.Collection.objects.get(id=collection_id) + collection = getattr(models, "Collection").objects.get(id=collection_id) scoped = self.apply_scoping(collection, defer=False) - elif isinstance(self.overrideScope['collection'], Callable): + elif callable(self.overrideScope['collection']): collection = self.overrideScope['collection'](self, row_index) if row_index is not None else default_collection scoped = self.apply_scoping(collection, defer=False) @@ -171,7 +171,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new # ScopedUploadTable - scoped_disambiguated = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped + scoped_disambiguated: Union[ScopedUploadable, ScopedUploadTable] = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped # type: ignore # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) From 4a2fef6ee15a5c358df781a666328a46e2fb59ad Mon Sep 17 00:00:00 2001 From: melton-jason Date: Sun, 9 Apr 2023 12:05:11 -0500 Subject: [PATCH 26/31] Cast type of `scoped` in method bind() to ScopedUploadTable --- specifyweb/workbench/upload/upload_table.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 841a8c622b7..6ee6c2ff3a3 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -1,7 +1,7 @@ import logging from functools import reduce -from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal +from typing import List, Dict, Any, NamedTuple, Union, Optional, Set, Callable, Literal, cast from django.db import transaction, IntegrityError @@ -169,9 +169,14 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # the record should be uploaded in the logged-in collection if scoped is None: scoped = self.apply_scoping(default_collection, defer=False) + # self.apply_scoping is annotated to Union["ScopedUploadTable", Any] + # But at this point we know the variable scoped will always be a ScopedUploadTable + # We tell typing the type of the variable scoped will be ScopedUploadTable with the cast() function + scoped = cast(ScopedUploadTable, scoped) + # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new # ScopedUploadTable - scoped_disambiguated: Union[ScopedUploadable, ScopedUploadTable] = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped # type: ignore + scoped_disambiguated = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) From 22895253172c4cdc21c879b62ab320e320f95fd8 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Sun, 9 Apr 2023 13:02:51 -0500 Subject: [PATCH 27/31] Ensure type of ScopedUploadTable is ScopedUploadTable after disambiguation --- specifyweb/workbench/upload/upload_table.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index 6ee6c2ff3a3..a4c6be9283a 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -176,7 +176,8 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new # ScopedUploadTable - scoped_disambiguated = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped + # Because ScopedUploadTable.disambiguate() has return type of ScopedUploadable, we must specify the type as ScopedUploadTable + scoped_disambiguated: ScopedUploadTable = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) From 7092b243a5e5e0a2544e8d0f5fcbd0a5bc8381c1 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Sun, 9 Apr 2023 13:26:15 -0500 Subject: [PATCH 28/31] Use typing.cast() rather than variable type assignment --- specifyweb/workbench/upload/upload_table.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specifyweb/workbench/upload/upload_table.py b/specifyweb/workbench/upload/upload_table.py index a4c6be9283a..f299853b3f5 100644 --- a/specifyweb/workbench/upload/upload_table.py +++ b/specifyweb/workbench/upload/upload_table.py @@ -177,7 +177,7 @@ def bind(self, default_collection, row: Row, uploadingAgentId: int, auditor: Aud # If the DeferredScope UploadTable contained any disambiguation data, then apply the disambiguation to the new # ScopedUploadTable # Because ScopedUploadTable.disambiguate() has return type of ScopedUploadable, we must specify the type as ScopedUploadTable - scoped_disambiguated: ScopedUploadTable = scoped.disambiguate(self.disambiguation) if self.disambiguation is not None else scoped + scoped_disambiguated = cast(ScopedUploadTable, scoped.disambiguate(self.disambiguation)) if self.disambiguation is not None else scoped # Finally bind the ScopedUploadTable and return the BoundUploadTable or ParseFailures return scoped_disambiguated.bind(default_collection, row, uploadingAgentId, auditor, cache, row_index) From a431c5b69545148a8424bc1781015939e62147ad Mon Sep 17 00:00:00 2001 From: melton-jason Date: Sun, 9 Apr 2023 13:34:56 -0500 Subject: [PATCH 29/31] Call UploadTestsBase setUp in ScopingTests setUp --- specifyweb/workbench/upload/tests/testscoping.py | 1 + 1 file changed, 1 insertion(+) diff --git a/specifyweb/workbench/upload/tests/testscoping.py b/specifyweb/workbench/upload/tests/testscoping.py index b4fdf156c6b..7e596396f70 100644 --- a/specifyweb/workbench/upload/tests/testscoping.py +++ b/specifyweb/workbench/upload/tests/testscoping.py @@ -10,6 +10,7 @@ class ScopingTests(UploadTestsBase): def setUp(self) -> None: + super().setUp() self.rel_type_name = "ToRightSide" self.right_side_collection = get_table('Collection').objects.create( From f45f6d2f164521232e0daed18524c532b5c26a61 Mon Sep 17 00:00:00 2001 From: melton-jason Date: Sun, 9 Apr 2023 13:50:30 -0500 Subject: [PATCH 30/31] Remove erroneous super().setUp() at end of ScopingTests setup() --- specifyweb/workbench/upload/tests/testscoping.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/specifyweb/workbench/upload/tests/testscoping.py b/specifyweb/workbench/upload/tests/testscoping.py index 7e596396f70..72c85faa7c2 100644 --- a/specifyweb/workbench/upload/tests/testscoping.py +++ b/specifyweb/workbench/upload/tests/testscoping.py @@ -67,9 +67,6 @@ def setUp(self) -> None: } } } - - - return super().setUp() def test_embedded_collectingevent(self) -> None: self.collection.isembeddedcollectingevent = True From 5fdc90bc63476329ee20b196412a0f47a657087d Mon Sep 17 00:00:00 2001 From: melton-jason Date: Thu, 18 May 2023 20:24:20 +0000 Subject: [PATCH 31/31] Lint code with ESLint and Prettier Triggered by c01fcd0aa1561bab248737028d236c18dea14eae on branch refs/heads/issue-3089 --- .../js_src/lib/components/FormFields/QueryComboBox.tsx | 2 +- specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx | 2 +- .../frontend/js_src/lib/components/FormSliders/RecordSet.tsx | 2 +- .../lib/components/Preferences/CollectionDefinitions.tsx | 3 ++- .../frontend/js_src/lib/components/Preferences/Editor.tsx | 4 ++-- .../frontend/js_src/lib/components/Preferences/Renderers.tsx | 4 ++-- specifyweb/frontend/js_src/lib/components/Security/User.tsx | 4 ++-- .../lib/components/WbPlanView/__tests__/automapper.test.ts | 2 +- 8 files changed, 12 insertions(+), 11 deletions(-) diff --git a/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx b/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx index 98521427efa..4078514add8 100644 --- a/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormFields/QueryComboBox.tsx @@ -162,7 +162,7 @@ export function QueryComboBox({ typeof resource.getDependentResource(field.name) === 'object') ? resource .rgetPromise(field.name) - .then((resource) => + .then(async (resource) => resource === undefined || resource === null ? { label: '' as LocalizedString, diff --git a/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx b/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx index c948e0ff0ed..48d82b3568a 100644 --- a/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormMeta/index.tsx @@ -18,6 +18,7 @@ import { PrintOnSave } from '../FormFields/Checkbox'; import type { ViewDescription } from '../FormParse'; import { SubViewContext } from '../Forms/SubView'; import { isTreeResource } from '../InitialContext/treeRanks'; +import { interactionTables } from '../Interactions/config'; import { Dialog } from '../Molecules/Dialog'; import { ProtectedAction, @@ -33,7 +34,6 @@ import { QueryTreeUsages } from './QueryTreeUsages'; import { ReadOnlyMode } from './ReadOnlyMode'; import { ShareRecord } from './ShareRecord'; import { SubViewMeta } from './SubViewMeta'; -import { interactionTables } from '../Interactions/config'; /** * Form preferences host context aware user preferences and other meta-actions. diff --git a/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx b/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx index 8c5535e95f0..ac3548fcd68 100644 --- a/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx +++ b/specifyweb/frontend/js_src/lib/components/FormSliders/RecordSet.tsx @@ -359,7 +359,7 @@ function RecordSet({ }).then(({ totalCount }) => totalCount !== 0), }) ) - ).then((results) => { + ).then(async (results) => { const [nonDuplicates, duplicates] = split( results, ({ isDuplicate }) => isDuplicate diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx index daf2f946dae..4a0a1c2a3f1 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/CollectionDefinitions.tsx @@ -6,7 +6,8 @@ import type { RA } from '../../utils/types'; import { ensure } from '../../utils/types'; import { error } from '../Errors/assert'; import type { StatLayout } from '../Statistics/types'; -import { GenericPreferences, defineItem } from './types'; +import type { GenericPreferences } from './types'; +import { defineItem } from './types'; export const collectionPreferenceDefinitions = { statistics: { diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx index b530c261058..ef95e189ebc 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/Editor.tsx @@ -1,11 +1,11 @@ import React from 'react'; +import { useLiveState } from '../../hooks/useLiveState'; +import type { AppResourceTab } from '../AppResources/TabDefinitions'; import { PreferencesContent } from '../Preferences'; import { BasePreferences } from '../Preferences/BasePreferences'; import { userPreferenceDefinitions } from '../Preferences/UserDefinitions'; import { userPreferences } from '../Preferences/userPreferences'; -import { AppResourceTab } from '../AppResources/TabDefinitions'; -import { useLiveState } from '../../hooks/useLiveState'; export const UserPreferencesEditor: AppResourceTab = function ({ isReadOnly, diff --git a/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx b/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx index ef782a81ecc..c20f4f0a773 100644 --- a/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx +++ b/specifyweb/frontend/js_src/lib/components/Preferences/Renderers.tsx @@ -31,9 +31,9 @@ import { rawMenuItemsPromise } from '../Header/menuItemDefinitions'; import { useMenuItems, useUserTools } from '../Header/menuItemProcessing'; import { AttachmentPicker } from '../Molecules/AttachmentPicker'; import { AutoComplete } from '../Molecules/AutoComplete'; -import { userPreferences } from './userPreferences'; import { ListEdit } from '../Toolbar/QueryTablesEdit'; -import { PreferenceItem, PreferenceItemComponent } from './types'; +import type { PreferenceItem, PreferenceItemComponent } from './types'; +import { userPreferences } from './userPreferences'; export const ColorPickerPreferenceItem: PreferenceItemComponent = function ColorPickerPreferenceItem({ diff --git a/specifyweb/frontend/js_src/lib/components/Security/User.tsx b/specifyweb/frontend/js_src/lib/components/Security/User.tsx index e343515cfe6..398ffc343f9 100644 --- a/specifyweb/frontend/js_src/lib/components/Security/User.tsx +++ b/specifyweb/frontend/js_src/lib/components/Security/User.tsx @@ -468,7 +468,7 @@ function UserView({ status: Http.NO_CONTENT, }) ) - .then(({ data, status }) => + .then(async ({ data, status }) => status === Http.BAD_REQUEST ? setState({ type: 'SettingAgents', @@ -515,7 +515,7 @@ function UserView({ }) : true ) - .then((canContinue) => + .then(async (canContinue) => canContinue === true ? Promise.all([ typeof password === 'string' && password !== '' diff --git a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts index 0e7101bd1b7..7aa22c6671a 100644 --- a/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts +++ b/specifyweb/frontend/js_src/lib/components/WbPlanView/__tests__/automapper.test.ts @@ -3,8 +3,8 @@ import { theories } from '../../../tests/utils'; import type { RA } from '../../../utils/types'; import type { AutoMapperResults } from '../autoMapper'; import { - AutoMapper as AutoMapperConstructor, type AutoMapperConstructorParameters, + AutoMapper as AutoMapperConstructor, circularTables, } from '../autoMapper';