diff --git a/requirements.txt b/requirements.txt index caa22a0..cd64248 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ -cript==0.6.1 -openpyxl==3.0.10 -pandas==1.5.1 -black==22.10.0 -pre-commit==2.20.0 -eel==0.15.0 -pyinstaller==5.6.2 +cript==0.6.3 +openpyxl==3.1.2 +pandas==2.0.1 +black==23.3.0 +pre-commit==3.3.2 +eel==0.16.0 +pyinstaller==5.11.0 diff --git a/src/python/create.py b/src/python/create.py index 788a591..fdc9673 100644 --- a/src/python/create.py +++ b/src/python/create.py @@ -6,40 +6,51 @@ row_input_can_start_from = 5 -def create_experiments(parsed_experiments, collection): - """Compiles a dictionary of cript Experiment objects. If a parsed experiment is able to be turned - into an Experiment object it is added to an experiments dictionary and that dictionary is returned. +def create_experiments_and_inventories(parsed_experiments, collection): + """Compiles dictionaries of CRIPT Experiment objects and CRIPT Inventory objects. If a parsed experiment/inventory is able to be turned + into an Experiment/Inventory object it is added to an experiments dictionary and that dictionary is returned. parsed_...-dict of dicts group-object collection-object returns- dict of objects""" experiments = {} + inventories = {} for key, parsed_experiment in parsed_experiments.items(): experiment_dict = {"collection": collection} - + inventory_dict = {"collection": collection} + inventory = False for parsed_cell in parsed_experiment.values(): if isinstance(parsed_cell, dict): cell_type = parsed_cell["type"] cell_key = parsed_cell["key"] cell_value = parsed_cell["value"] - # Only attribute types should be in Experiment + # Only attribute and that specific identifier should be in experiment if cell_type == "attribute": experiment_dict[cell_key] = cell_value + inventory_dict[cell_key] = cell_value + elif cell_type == "identifier": + if cell_key == "Experiment or Inventory": + if cell_value.lower() == "i": + inventory = True + if inventory: + invObj = _create_object(cript.Inventory, inventory_dict, parsed_cell) + if invObj is not None: + inventories[key] = invObj + else: + experiment = _create_object(cript.Experiment, experiment_dict, parsed_cell) + # Only adds Experiment objects + if experiment is not None: + experiments[key] = experiment - experiment = _create_object(cript.Experiment, experiment_dict, parsed_cell) - # Only adds Experiment objects - if experiment is not None: - experiments[key] = experiment - - return experiments + return experiments, inventories def create_citations(parsed_citations, group): """Compiles dictionaries with Data and File cript objects. parsed_...-dict of dicts - group-obj - returns-tuple of dicts of objs + group-cript Group node + returns-tuple of dicts of reference nodes and citation nodes """ references = {} @@ -135,6 +146,7 @@ def create_materials(parsed_materials, project, data, citations): citations-list return-dict of obj""" materials = {} + inventory_dict = {} for key, parsed_material in parsed_materials.items(): use_existing = False @@ -143,6 +155,8 @@ def create_materials(parsed_materials, project, data, citations): "identifiers": [], "properties": [], } + belongs_in_inv = False + inv_name = None for parsed_cell in parsed_material.values(): cell_type = parsed_cell["type"] @@ -162,18 +176,24 @@ def create_materials(parsed_materials, project, data, citations): elif cell_type == "property": if parsed_cell["key"] == "use_existing": - use_existing = cellToBool(parsed_cell["value"]) + use_existing = is_cell_true(parsed_cell["value"]) continue property = _create_property(parsed_cell, data, citations) material_dict["properties"].append(property) + elif cell_type == "relation": + belongs_in_inv = True + inv_name = cell_value # Add characteristics to an already created material node if use_existing: try: # try to get the material using its name - name_ = parsed_material["name"]["value"] - material = cript.Material.get(name=name_, project=project.uid) + mat_name = parsed_material["name"]["value"] + new_project = cript.Project.get( + name=parsed_material["use_existing"]["value"] + ) + material = cript.Material.get(name=mat_name, project=new_project.uid) # If there is a get error add it to the errors sheet except ValueError as e: @@ -185,22 +205,39 @@ def create_materials(parsed_materials, project, data, citations): # If the material had a successful GET request, add properties, identifiers, # and select attributes as written in the excel else: + + material = copyMaterial(material, new_project, project) + + # Add properties,identifiers, and attributes to material for property in material_dict["properties"]: material.add_property(property) for identifier in material_dict["identifiers"]: material.add_identifier(identifier) for key_ in material_dict: if key_ == "keywords": - material.keywords += material_dict["keywords"] + if material.keywords is not None: + material.keywords += material_dict["keywords"] + else: + material.keywords = material_dict["keywords"] elif key_ == "notes": - material.notes += material_dict["notes"] + if material.notes is not None: + material.notes += material_dict["notes"] + else: + material.notes = material_dict["notes"] + # create new material object otherwise else: material = _create_object(cript.Material, material_dict, parsed_cell) if material is not None: + materials[key] = material + if belongs_in_inv: + if inventory_dict.get(cell_value, None): + inventory_dict[inv_name].append(material) + else: + inventory_dict[inv_name] = [material] - return materials + return materials, inventory_dict def create_mixtures(parsed_components, materials): @@ -433,7 +470,7 @@ def _create_property(parsed_property, data, citations): property_dict["conditions"].append(condition) elif cell_type == "method": - if cellToBool(parsed_cell["value"]): + if is_cell_true(parsed_cell["value"]): property_dict["method"] = parsed_cell["key"] elif cell_type == "relation": @@ -530,6 +567,37 @@ def _get_relation(related_objs, cell_value, parsed_cell): return None -def cellToBool(val): +def is_cell_true(val): """Converts a cell value to a useable boolean""" - return True if str(val).lower() == "true" else False + return str(val).lower() != "false" + + +def copyMaterial(material, new_project, project): + """ + Takes a material node and adjusts values to get rid of legacy code and incompatible features + inputs: + material - cript material node + new_project - cript project node + project - cript project node + + returns - cript material node + """ + if new_project.name != project.name: + # Sets new project and gets rid of url and uid to make new node object + material.project = project + material.url = None + material.uid = None + # Gets rid of citations that would cause permissions errors + if material.group.name != project.group.name: + for property in material.properties: + property.citations = [] + material.group = project.group + + newProperties = [] + # Gets rid of any legacy properties/custom that won't upload + for property in material.properties: + if "+" not in property.key: + newProperties.append(property) + material.properties = newProperties + + return material diff --git a/src/python/excel_uploader_main.py b/src/python/excel_uploader_main.py index df73b1d..b0b2287 100644 --- a/src/python/excel_uploader_main.py +++ b/src/python/excel_uploader_main.py @@ -186,8 +186,8 @@ def upload_driver(self, excel_file_path, gui_object): # Create and validate ### - experiments = create.create_experiments( - parsed_sheets["experiment"], self.collection_object + experiments, inventories = create.create_experiments_and_inventories( + parsed_sheets["experiment & inventory"], self.collection_object ) references, citations = create.create_citations( parsed_sheets["citation"], self.project_object.group @@ -195,7 +195,7 @@ def upload_driver(self, excel_file_path, gui_object): data, files = create.create_data( parsed_sheets["data"], self.project_object, experiments, citations ) - materials = create.create_materials( + materials, inv_dict = create.create_materials( parsed_sheets["material"], self.project_object, data, citations ) materials = create.create_mixtures( @@ -256,6 +256,10 @@ def upload_driver(self, excel_file_path, gui_object): upload.upload(references, "Reference", self, gui_object) + # Reassigns saved file nodes into their corresponding unsaved data nodes + for key, file in files.items(): + data[key].files[0] = file + upload.upload(data, "Data", self, gui_object) upload.upload(materials, "Material", self, gui_object) @@ -266,6 +270,13 @@ def upload_driver(self, excel_file_path, gui_object): parsed_sheets["data"], data, processes, self.api, self, gui_object ) + # Add saved material nodes to inventories and save inventories + for name, mat_arr in inv_dict.items(): + inventory = inventories[name] + for mat in mat_arr: + inventory.add_material(mat) + upload.upload(inventories, "Inventory", self, gui_object) + return self.error_list def get_collections_url(self): diff --git a/src/python/parse.py b/src/python/parse.py index a848b35..27789d6 100644 --- a/src/python/parse.py +++ b/src/python/parse.py @@ -43,13 +43,15 @@ def parse(self): cell_info = self._get_cell_info(index, row, column) # Check if column should be skipped - if self._skip_column(column[1], cell_info["value"], cell_info["type"]): + if self._skip_column( + cell_info["key"], cell_info["value"], cell_info["type"] + ): continue # Convert list values (with ";" separator) to Python lists # Manually skip fields commonly containing semicolons if ( - cell_info["value"] not in ("notes", "description") + cell_info["key"] not in ("notes", "description") and isinstance(cell_info["value"], str) and ";" in cell_info["value"] ): diff --git a/src/python/sheet_parameters.py b/src/python/sheet_parameters.py index f969272..7ac3193 100644 --- a/src/python/sheet_parameters.py +++ b/src/python/sheet_parameters.py @@ -1,7 +1,7 @@ # Define sheet parameters sheet_parameters = [ { - "name": "experiment", + "name": "experiment & inventory", "required_columns": ("name",), "unique_columns": ("name",), }, diff --git a/src/python/upload.py b/src/python/upload.py index 0abe578..aaebba2 100644 --- a/src/python/upload.py +++ b/src/python/upload.py @@ -72,6 +72,10 @@ def upload(obj_dict, obj_type, excel_uploader_object, gui_object): continue else: raise error + except cript.api.exceptions.APIError as error: + obj_dict[key] = cript.File.get( + name=obj.name, project=excel_uploader_object.project_object.uid + ) # TODO this needs specific errors instead of a catch all except Exception as error: