diff --git a/src/python/create.py b/src/python/create.py index 788a591..c4289ed 100644 --- a/src/python/create.py +++ b/src/python/create.py @@ -6,40 +6,51 @@ row_input_can_start_from = 5 -def create_experiments(parsed_experiments, collection): - """Compiles a dictionary of cript Experiment objects. If a parsed experiment is able to be turned - into an Experiment object it is added to an experiments dictionary and that dictionary is returned. +def create_experiments_and_inventories(parsed_experiments, collection): + """Compiles dictionaries of CRIPT Experiment objects and CRIPT Inventory objects. If a parsed experiment/inventory is able to be turned + into an Experiment/Inventory object it is added to an experiments dictionary and that dictionary is returned. parsed_...-dict of dicts group-object collection-object returns- dict of objects""" experiments = {} + inventories = {} for key, parsed_experiment in parsed_experiments.items(): experiment_dict = {"collection": collection} - + inventory_dict = {"collection": collection} + inventory = False for parsed_cell in parsed_experiment.values(): if isinstance(parsed_cell, dict): cell_type = parsed_cell["type"] cell_key = parsed_cell["key"] cell_value = parsed_cell["value"] - # Only attribute types should be in Experiment + # Only attribute and that specific identifier should be in experiment if cell_type == "attribute": experiment_dict[cell_key] = cell_value + inventory_dict[cell_key] = cell_value + elif cell_type == "identifier": + if cell_key == "Experiment or Inventory": + if cell_value.lower() == "i": + inventory = True + if inventory: + invObj = _create_object(cript.Inventory, inventory_dict, parsed_cell) + if invObj is not None: + inventories[key] = invObj + else: + experiment = _create_object(cript.Experiment, experiment_dict, parsed_cell) + # Only adds Experiment objects + if experiment is not None: + experiments[key] = experiment - experiment = _create_object(cript.Experiment, experiment_dict, parsed_cell) - # Only adds Experiment objects - if experiment is not None: - experiments[key] = experiment - - return experiments + return experiments, inventories def create_citations(parsed_citations, group): """Compiles dictionaries with Data and File cript objects. parsed_...-dict of dicts - group-obj - returns-tuple of dicts of objs + group-cript Group node + returns-tuple of dicts of reference nodes and citation nodes """ references = {} @@ -78,7 +89,7 @@ def create_data(parsed_data, project, experiments, citations): for key, parsed_datum in parsed_data.items(): datum_dict = {"files": [], "citations": []} - + files_list = [] for parsed_cell in parsed_datum.values(): if isinstance(parsed_cell, dict): cell_type = parsed_cell["type"] @@ -89,6 +100,18 @@ def create_data(parsed_data, project, experiments, citations): if cell_key == "source": # Grab File object source file_source = cell_value + files_list.append( + _create_object( + cript.File, + { + "project": project, + "source": file_source, + "type": "data", + }, + parsed_cell, + ) + ) + continue else: datum_dict[cell_key] = cell_value @@ -107,21 +130,13 @@ def create_data(parsed_data, project, experiments, citations): elif cell_key == "sample_preparation": datum_dict["sample_preparation"] = None - file = _create_object( - cript.File, - { - "project": project, - "source": file_source, - "type": "data", - }, - parsed_cell, - ) - datum_dict["files"].append(file) + if None not in files_list: + files[key] = files_list + datum = _create_object(cript.Data, datum_dict, parsed_cell) - if None not in (datum, file): + if datum: data[key] = datum - files[key] = file return data, files @@ -135,6 +150,7 @@ def create_materials(parsed_materials, project, data, citations): citations-list return-dict of obj""" materials = {} + inventory_dict = {} for key, parsed_material in parsed_materials.items(): use_existing = False @@ -143,6 +159,8 @@ def create_materials(parsed_materials, project, data, citations): "identifiers": [], "properties": [], } + belongs_in_inv = False + inv_name = None for parsed_cell in parsed_material.values(): cell_type = parsed_cell["type"] @@ -150,7 +168,13 @@ def create_materials(parsed_materials, project, data, citations): cell_value = parsed_cell["value"] if cell_type == "attribute": - material_dict[cell_key] = cell_value + + if cell_key == "computational_forcefield": + material_dict[cell_key] = create_computational_forcefield( + parsed_cell + ) + else: + material_dict[cell_key] = cell_value elif cell_type == "identifier": identifier = _create_object( @@ -162,18 +186,24 @@ def create_materials(parsed_materials, project, data, citations): elif cell_type == "property": if parsed_cell["key"] == "use_existing": - use_existing = cellToBool(parsed_cell["value"]) + use_existing = is_cell_true(parsed_cell["value"]) continue property = _create_property(parsed_cell, data, citations) material_dict["properties"].append(property) + elif cell_type == "relation": + belongs_in_inv = True + inv_name = cell_value # Add characteristics to an already created material node if use_existing: try: # try to get the material using its name - name_ = parsed_material["name"]["value"] - material = cript.Material.get(name=name_, project=project.uid) + mat_name = parsed_material["name"]["value"] + new_project = cript.Project.get( + name=parsed_material["use_existing"]["value"] + ) + material = cript.Material.get(name=mat_name, project=new_project.uid) # If there is a get error add it to the errors sheet except ValueError as e: @@ -185,22 +215,39 @@ def create_materials(parsed_materials, project, data, citations): # If the material had a successful GET request, add properties, identifiers, # and select attributes as written in the excel else: + + material = copyMaterial(material, new_project, project) + + # Add properties,identifiers, and attributes to material for property in material_dict["properties"]: material.add_property(property) for identifier in material_dict["identifiers"]: material.add_identifier(identifier) for key_ in material_dict: if key_ == "keywords": - material.keywords += material_dict["keywords"] + if material.keywords is not None: + material.keywords += material_dict["keywords"] + else: + material.keywords = material_dict["keywords"] elif key_ == "notes": - material.notes += material_dict["notes"] + if material.notes is not None: + material.notes += material_dict["notes"] + else: + material.notes = material_dict["notes"] + # create new material object otherwise else: material = _create_object(cript.Material, material_dict, parsed_cell) if material is not None: + materials[key] = material + if belongs_in_inv: + if inventory_dict.get(cell_value, None): + inventory_dict[inv_name].append(material) + else: + inventory_dict[inv_name] = [material] - return materials + return materials, inventory_dict def create_mixtures(parsed_components, materials): @@ -236,6 +283,248 @@ def create_mixtures(parsed_components, materials): return materials +def create_computation( + parsed_computations, experiments, data, citations, software_configurations +): + """Creates a dictionary of Computation objects based on the parsed inputs + params: + @parsed_computations dictionary of dictionaries containing information about computation objects + @experiments dictionary of experiment objects + @data dictionary of data objects + @citations dictionary of citation objects + @software_configurations dictionary of software configuration objects + + returns: + dictionary of Computation objects + """ + computations = {} + + for key, parsed_process in parsed_computations.items(): + comp_dict = {"conditions": [], "software_configurations": []} + + for parsed_cell in parsed_process.values(): + cell_type = parsed_cell["type"] + cell_key = parsed_cell["key"] + cell_value = parsed_cell["value"] + + if cell_type == "attribute": + if cell_key == "citations": + pass + else: + comp_dict[cell_key] = cell_value + + # Gets existing object if available + elif cell_type == "relation": + if cell_key == "experiment": + comp_dict["experiment"] = _get_relation( + experiments, cell_value, parsed_cell + ) + elif "software_configuration" in cell_key: + software_config = _get_relation( + software_configurations, cell_value, parsed_cell + ) + if software_config: + comp_dict["software_configurations"].append(software_config) + + elif cell_type == "condition": + condition = _create_condition(parsed_cell, data, citations) + comp_dict["conditions"].append(condition) + + computation = _create_object(cript.Computation, comp_dict, parsed_cell) + if computation is not None: + computations[key] = computation + + return computations + + +def create_prerequisite_computation(parsed_prerequisites, computations): + """Attaches prerequisite Computation to a Computation node. + params: + parsed_...-dict of dicts + processes-dict of objs + + returns: + void + """ + for key, parsed_prerequisite in parsed_prerequisites.items(): + for parsed_cell in parsed_prerequisite.values(): + cell_type = parsed_cell["type"] + cell_key = parsed_cell["key"] + cell_value = parsed_cell["value"] + + if cell_type == "relation": + if cell_key == "computation": + computation = _get_relation(computations, cell_value, parsed_cell) + + elif cell_key == "prerequisite": + prerequisite = _get_relation(computations, cell_value, parsed_cell) + + if None not in (computation, prerequisite): + computation.prerequisite_computation = prerequisite + + +def create_computational_process( + parsed_comp_processes, experiments, software_configurations, data, citations +): + """Creates a dictionary of Computational_process objects to be returned. + params: + @parsed_comp_processes dict of dict of computational process information + @experiments -dict of Experiment objects + @data dict of data objects + @citations dict of citation objects + returns + dict of objects + """ + comp_processes = {} + + for key, parsed_comp_process in parsed_comp_processes.items(): + comp_process_dict = { + "properties": [], + "conditions": [], + "software_configurations": [], + } + + for parsed_cell in parsed_comp_process.values(): + cell_type = parsed_cell["type"] + cell_key = parsed_cell["key"] + cell_value = parsed_cell["value"] + + if cell_type == "attribute": + comp_process_dict[cell_key] = cell_value + + # Gets existing object if available + elif cell_type == "relation": + if cell_key == "experiment": + comp_process_dict["experiment"] = _get_relation( + experiments, cell_value, parsed_cell + ) + elif "software_configuration" in cell_key: + software_config = _get_relation( + software_configurations, cell_value, parsed_cell + ) + if software_config: + comp_process_dict["software_configurations"].append( + software_config + ) + + # Creates objects that will go into process node + elif cell_type == "property": + property = _create_property(parsed_cell, data, citations) + comp_process_dict["properties"].append(property) + + elif cell_type == "condition": + condition = _create_condition(parsed_cell, data, citations) + comp_process_dict["conditions"].append(condition) + + comp_process = _create_object( + cript.ComputationalProcess, comp_process_dict, parsed_cell + ) + if comp_process is not None: + comp_processes[key] = comp_process + + return comp_processes + + +def create_software_configuration(parsed_software, citations, project): + """Creates a dictionary of Software_Configuration objects based on the parsed inputs + params: + @parsed_computations dictionary of dictionaries containing information about software configuration objects + @citations dictionary of citation objects + + returns: + dictionary of software_configuration objects + """ + software_configurations = {} + for key, parsed_process in parsed_software.items(): + config_dict = { + "software": None, + "algorithms": [], + } # dictionary for Software Configuartion object + software_dict = {"group": project.group} # dictionary for Software object + for parsed_cell in parsed_process.values(): + cell_type = parsed_cell["type"] + cell_key = parsed_cell["key"] + cell_value = parsed_cell["value"] + + if cell_type == "attribute": + if cell_key == "citations": + continue + elif cell_key in {"name", "version", "source"}: + software_dict[cell_key] = cell_value + elif cell_key == "notes": + config_dict[cell_key] = cell_value + + # create algorithm + elif isinstance(cell_type, dict): + + if cell_type["type"] == "attribute": + alg_obj = create_algorithm(parsed_cell) + if alg_obj: + config_dict["algorithms"].append(alg_obj) + + software = _create_object(cript.Software, software_dict, parsed_cell) + if software: + try: + software.save() + except: + software = cript.Software.get( + name=software_dict["name"], + version=software_dict["version"], + ) + + config_dict["software"] = software + + # Take algorithms out if none are present + if not config_dict["algorithms"]: + config_dict.pop("algorithms") + + software_configuration = _create_object( + cript.SoftwareConfiguration, config_dict, parsed_cell + ) + if software_configuration is not None: + software_configurations[key] = software_configuration + + return software_configurations + + +def create_in_out_data_connections( + parsed_in_out_data, computations, computational_processes, data +): + """ + Attaches input and output data to a Computation or Computatinal Process object + params: + @parsed_in_out_data dict of dicts of information relating data to another node + @computations dict of computation objects + @computational_processes dict of computational process objects + returns: + void + """ + merged_dict = ( + computations | computational_processes + ) # merge dictionaries for ease of access + + for key, parsed_info in parsed_in_out_data.items(): + input_data = None + output_data = None + for parsed_cell in parsed_info.values(): + cell_type = parsed_cell["type"] + cell_key = parsed_cell["key"] + cell_value = parsed_cell["value"] + + if cell_type == "relation": + if cell_key == "computation or computational process": + cript_obj = _get_relation(merged_dict, cell_value, parsed_cell) + elif cell_key == "input data": + input_data = _get_relation(data, cell_value, parsed_cell) + elif cell_key == "output data": + output_data = _get_relation(data, cell_value, parsed_cell) + + if None not in (cript_obj, input_data): + cript_obj.input_data.append(input_data) + if None not in (cript_obj, output_data): + cript_obj.output_data.append(output_data) + + def create_processes(parsed_processes, experiments, data, citations): """Creates a dictionary of Process objects to be returned. parsed_...-dict of objects @@ -279,7 +568,7 @@ def create_processes(parsed_processes, experiments, data, citations): return processes -def create_prerequisites(parsed_prerequisites, processes): +def create_prerequisite_process(parsed_prerequisites, processes): """Attaches prerequisite process information to a Process node. parsed_...-dict of dicts processes-dict of objs""" @@ -401,6 +690,36 @@ def create_equipment(parsed_equipment, processes, data, citations): process.equipment.append(piece) +def create_algorithm(parsed_cell): + """ + Auxilliary function to abstract the creation of an Algorithm object with Parameters + input: + @parsed_cell dictionary of information about Algorithm + returns: + algorithm object or None + + """ + + cell_type = parsed_cell["type"] + cell_value = parsed_cell["value"] + + alg_dict = {"parameters": []} + alg_dict["type"] = cell_type["value"] + alg_dict["key"] = cell_value + for key2, param in parsed_cell.items(): + if "parameter" in key2: + param_dict = {} + param_dict["key"] = param["value"] + param_dict["value"] = param["input"]["value"] + param_dict["unit"] = param.get("unit") + param_object = _create_object(cript.Parameter, param_dict, parsed_cell) + if param_object: + alg_dict["parameters"].append(param_object) + + alg_obj = _create_object(cript.Algorithm, alg_dict, parsed_cell) + return alg_obj + + def _create_property(parsed_property, data, citations): """Tries to create a Property object that contains plain attributes as well as other objects within. Returns Property object or None. @@ -433,7 +752,7 @@ def _create_property(parsed_property, data, citations): property_dict["conditions"].append(condition) elif cell_type == "method": - if cellToBool(parsed_cell["value"]): + if is_cell_true(parsed_cell["value"]): property_dict["method"] = parsed_cell["key"] elif cell_type == "relation": @@ -530,6 +849,82 @@ def _get_relation(related_objs, cell_value, parsed_cell): return None -def cellToBool(val): +def is_cell_true(val): """Converts a cell value to a useable boolean""" - return True if str(val).lower() == "true" else False + return str(val).lower() != "false" + + +def copyMaterial(material, new_project, project): + """ + Takes a material node and adjusts values to get rid of legacy code and incompatible features + inputs: + material - cript material node + new_project - cript project node + project - cript project node + + returns - cript material node + """ + if new_project.name != project.name: + # Sets new project and gets rid of url and uid to make new node object + material.project = project + material.url = None + material.uid = None + # Gets rid of citations that would cause permissions errors + if material.group.name != project.group.name: + for property in material.properties: + property.citations = [] + material.group = project.group + + newProperties = [] + # Gets rid of any legacy properties/custom that won't upload + for property in material.properties: + if "+" not in property.key: + newProperties.append(property) + material.properties = newProperties + + return material + +def create_computational_forcefield(parsed_cell): + """ + Create a computational forcefield object + param: + @parsed_cell dictionary of information about object + returns: + Computational_forcefield object or None + """ + # Try to assign all of computational_forcefield's attributes + # Can't assign data and citation here + building_block = ( + temp_dict.get("value") + if (temp_dict := parsed_cell.get("building_block")) + else None + ) + coarse_grained_mapping = ( + temp_dict.get("value") + if (temp_dict := parsed_cell.get("coarse_grained_mapping")) + else None + ) + implicit_solvent = ( + temp_dict.get("value") + if (temp_dict := parsed_cell.get("implicit_solvent")) + else None + ) + source = ( + temp_dict.get("value") if (temp_dict := parsed_cell.get("source")) else None + ) + description = ( + temp_dict.get("value") + if (temp_dict := parsed_cell.get("description")) + else None + ) + + object_dict = { + "key": parsed_cell["value"], + "building_block": building_block, + "coarse_grained_mapping": coarse_grained_mapping, + "implicit_solvent": implicit_solvent, + "source": source, + "description": description, + } + return _create_object(cript.ComputationalForcefield, object_dict, parsed_cell) + diff --git a/src/python/excel_uploader_main.py b/src/python/excel_uploader_main.py index df73b1d..008fc76 100644 --- a/src/python/excel_uploader_main.py +++ b/src/python/excel_uploader_main.py @@ -90,12 +90,13 @@ def is_uploading_local_files(self, files): if len(files) < 1: return False - for key in files: - # if file starts with http or https, then file is local, and returns true - if not files[key].source.startswith("http://") and not files[ - key - ].source.startswith("https://"): - return True + for key, values in files.items(): + for ix in range(len(values)): + # if file starts with http or https, then file is local, and returns true + if not files[key][ix].source.startswith("http://") and not files[key][ + ix + ].source.startswith("https://"): + return True return False @@ -186,8 +187,8 @@ def upload_driver(self, excel_file_path, gui_object): # Create and validate ### - experiments = create.create_experiments( - parsed_sheets["experiment"], self.collection_object + experiments, inventories = create.create_experiments_and_inventories( + parsed_sheets["experiment & inventory"], self.collection_object ) references, citations = create.create_citations( parsed_sheets["citation"], self.project_object.group @@ -195,7 +196,7 @@ def upload_driver(self, excel_file_path, gui_object): data, files = create.create_data( parsed_sheets["data"], self.project_object, experiments, citations ) - materials = create.create_materials( + materials, inv_dict = create.create_materials( parsed_sheets["material"], self.project_object, data, citations ) materials = create.create_mixtures( @@ -205,6 +206,33 @@ def upload_driver(self, excel_file_path, gui_object): parsed_sheets["process"], experiments, data, citations ) + software_configurations = create.create_software_configuration( + parsed_sheets["software configuration"], citations, self.project_object + ) + + computations = create.create_computation( + parsed_sheets["computation"], + experiments, + data, + citations, + software_configurations, + ) + + computational_processes = create.create_computational_process( + parsed_sheets["computational process"], + experiments, + software_configurations, + data, + citations, + ) + + create.create_in_out_data_connections( + parsed_sheets["input & output data"], + computations, + computational_processes, + data, + ) + # if there is local files to upload, and they have not authenticated with storage client yet # take them to authenticate with globus if ( @@ -217,9 +245,19 @@ def upload_driver(self, excel_file_path, gui_object): return # create - create.create_prerequisites(parsed_sheets["prerequisite process"], processes) + + create.create_prerequisite_process( + parsed_sheets["prerequisite process"], processes + ) + + create.create_prerequisite_computation( + parsed_sheets["prerequisite computation"], computations + ) + + merged_processes = processes | computational_processes + create.create_ingredients( - parsed_sheets["process ingredient"], processes, materials + parsed_sheets["process ingredient"], merged_processes, materials ) create.create_products(parsed_sheets["process product"], processes, materials) create.create_equipment( @@ -256,16 +294,34 @@ def upload_driver(self, excel_file_path, gui_object): upload.upload(references, "Reference", self, gui_object) + # Reassigns saved file nodes into their corresponding unsaved data nodes + for key, vals in files.items(): + for file in vals: + data[key].files.append(file) + upload.upload(data, "Data", self, gui_object) upload.upload(materials, "Material", self, gui_object) + upload.upload(computations, "Computation", self, gui_object) + + upload.upload( + computational_processes, "Computational Processes", self, gui_object + ) + upload.upload(processes, "Process", self, gui_object) upload.add_sample_preparation_to_process( parsed_sheets["data"], data, processes, self.api, self, gui_object ) + # Add saved material nodes to inventories and save inventories + for name, mat_arr in inv_dict.items(): + inventory = inventories[name] + for mat in mat_arr: + inventory.add_material(mat) + upload.upload(inventories, "Inventory", self, gui_object) + return self.error_list def get_collections_url(self): diff --git a/src/python/parse.py b/src/python/parse.py index a848b35..27789d6 100644 --- a/src/python/parse.py +++ b/src/python/parse.py @@ -43,13 +43,15 @@ def parse(self): cell_info = self._get_cell_info(index, row, column) # Check if column should be skipped - if self._skip_column(column[1], cell_info["value"], cell_info["type"]): + if self._skip_column( + cell_info["key"], cell_info["value"], cell_info["type"] + ): continue # Convert list values (with ";" separator) to Python lists # Manually skip fields commonly containing semicolons if ( - cell_info["value"] not in ("notes", "description") + cell_info["key"] not in ("notes", "description") and isinstance(cell_info["value"], str) and ";" in cell_info["value"] ): diff --git a/src/python/sheet_parameters.py b/src/python/sheet_parameters.py index f969272..433f236 100644 --- a/src/python/sheet_parameters.py +++ b/src/python/sheet_parameters.py @@ -1,7 +1,7 @@ # Define sheet parameters sheet_parameters = [ { - "name": "experiment", + "name": "experiment & inventory", "required_columns": ("name",), "unique_columns": ("name",), }, @@ -50,4 +50,29 @@ "required_columns": ("process", "key"), "unique_columns": ("process", "key"), }, + { + "name": "computation", + "required_columns": ("experiment", "name", "type"), + "unique_columns": ("name"), + }, + { + "name": "prerequisite computation", + "required_columns": ("computation", "prerequisite"), + "unique_columns": ("computation", "prerequisite"), + }, + { + "name": "computational process", + "required_columns": ("experiment", "name", "type"), + "unique_columns": ("name"), + }, + { + "name": "software configuration", + "required_columns": ("name", "version"), + "unique_columns": ("name"), + }, + { + "name": "input & output data", + "required_columns": ("computation or computational process", "input data"), + "unique_columns": ("computation or computational process", "input data"), + }, ] diff --git a/src/python/upload.py b/src/python/upload.py index 0abe578..0f3238c 100644 --- a/src/python/upload.py +++ b/src/python/upload.py @@ -54,8 +54,20 @@ def upload(obj_dict, obj_type, excel_uploader_object, gui_object): for key, obj in obj_dict.items(): try: - if obj_type == "File" and obj.name is None: - obj.name = obj.source + if obj_type == "File": + # iterate through list of files + for file in obj: + if file.name is None: + file.name = file.source + # nested try and except to handle duplicate file issues + try: + file.save(update_existing=True) + except cript.api.exceptions.APIError as error: + obj_dict[key] = cript.File.get( + name=file.name, + project=excel_uploader_object.project_object.uid, + ) + continue obj.save(update_existing=True) @@ -72,6 +84,10 @@ def upload(obj_dict, obj_type, excel_uploader_object, gui_object): continue else: raise error + except cript.api.exceptions.APIError as error: + obj_dict[key] = cript.File.get( + name=obj.name, project=excel_uploader_object.project_object.uid + ) # TODO this needs specific errors instead of a catch all except Exception as error: diff --git a/tests/GUI/test_gui.py b/tests/GUI/test_gui.py new file mode 100644 index 0000000..ccade2d --- /dev/null +++ b/tests/GUI/test_gui.py @@ -0,0 +1,42 @@ +from selenium import webdriver +from selenium.webdriver.chrome.options import Options +from selenium.webdriver.common.by import By +import subprocess +import time + +from selenium.webdriver.common.by import By + +# Specify the path to the ChromeDriver executable +chrome_driver_path = r'C:\Program Files\Google\Chrome\Application\chrome.exe' + +# Start the Chrome browser in app mode +driver = webdriver.Chrome() + +driver.maximize_window() + +driver.get("http://localhost:8001/templates/base.html") + +# fill out the form +element = driver.find_element(By.ID, "host-input") +element.send_keys('criptapp.org') + +element = driver.find_element(By.ID, "api-token-input") +element.send_keys('this is my super secret token') + +element = driver.find_element(By.ID, "project-name") +element.send_keys('my project name') + +element = driver.find_element(By.ID, "collection-name") +element.send_keys('my collection') + +element = driver.find_element(By.ID, "excel-file-path") +driver.execute_script("arguments[0].removeAttribute('disabled');", element) +element.send_keys('hello world!') + +element = driver.find_element(By.ID, "upload-button") + +# submit the form +element.click() + +# wait and do not close browser so we can see what happened for a bit +time.sleep(1000)