"+k+" = "+str(self.option_handler.GetOptimizerOptions()[k])+"
\n" tmp_str+="feats = "+self.option_handler.feat_str +"
\n" @@ -483,7 +506,7 @@ def FourthStep(self,args={}): tmp_list=[] for t in self.error_comps: for c in t: - if self.option_handler.type[-1]!='features': + if self.option_handler.type[-1]!='features' and self.option_handler.type[-1]!= 'hippounit': #tmp_str.append( "*".join([str(c[0]),c[1].__name__])) tmp_list.append([self.ffun_mapper[c[1].__name__], str(c[2]), @@ -507,7 +530,7 @@ def FourthStep(self,args={}): tmp[1]+=c[t_idx][2] tmp[2]=c[t_idx][0] tmp[3]+=c[t_idx][2]*c[t_idx][0] - if self.option_handler.type[-1]!='features': + if self.option_handler.type[-1]!='features' and self.option_handler.type[-1]!= 'hippounit': tmp[0]=self.ffun_mapper[c[t_idx][1].__name__] else: tmp[0]=(c[t_idx][1]) @@ -525,8 +548,11 @@ def FourthStep(self,args={}): target_dict = {"data_type":self.option_handler.type[-1],"file_name":self.option_handler.input_dir.split('/')[-1],"number_of_traces":k_range,"stim_delay":self.option_handler.stim_del, "stim_duration":self.option_handler.stim_dur} json_var = {"model":self.name,"optimization":opt_dict,"parameters":param_dict,"error_function":error_dict, "algorithm":alg_dict,"target_data":target_dict} - - if self.option_handler.type[-1]=='features': + + if self.option_handler.type[-1]=='hippounit': + with open('metadata.json', 'w+') as outfile: + json.dump(json_var, outfile, indent=4) + elif self.option_handler.type[-1]=='features': with open(self.option_handler.input_dir, 'r') as outfile: input_features=json.load(outfile) amp_dict={amp_vals:[] for amp_vals in input_features["stimuli"]["amplitudes"]} diff --git a/optimizer/cmd_line.py b/optimizer/cmd_line.py index 1b9a4b27..bf15e955 100755 --- a/optimizer/cmd_line.py +++ b/optimizer/cmd_line.py @@ -22,7 +22,7 @@ def main(fname, param=None): """ try: with open(fname, "r") as f: - json_data = json.load(f) + json_data = json.load(f) except IOError as ioe: print(ioe) sys.exit("File not found!\n") @@ -46,45 +46,46 @@ def main(fname, param=None): core.ThirdStep(kwargs) core.FourthStep() print("resulting parameters: ", core.optimal_params) - fig = figure(1, figsize=(7, 6)) + fig = figure(figsize=(7, 6)) axes = fig.add_subplot(111) exp_data = [] model_data = [] - if core.option_handler.type[-1] != 'features': - for n in range(core.data_handler.number_of_traces()): - exp_data.extend(core.data_handler.data.GetTrace(n)) - model_data.extend(core.final_result[n]) - no_traces = core.data_handler.number_of_traces() - else: - for n in range(len(core.data_handler.features_data["stim_amp"])): - model_data.extend(core.final_result[n]) - no_traces = len(core.data_handler.features_data["stim_amp"]) - if core.option_handler.type[-1] != 'features': - t = int(ceil(core.option_handler.input_length)) - else: - t = int(ceil(core.option_handler.run_controll_tstop)) - step = core.option_handler.run_controll_dt - axes.set_xticks([n for n in range(0, int((t * no_traces) / (step)), int((t * no_traces) / (step) / 5.0)) ]) - axes.set_xticklabels([str(n) for n in range(0, int(t * no_traces), int((t * no_traces) / 5))]) + if core.option_handler.type[-1] != 'hippounit': + if core.option_handler.type[-1] != 'features': + for n in range(core.data_handler.number_of_traces()): + exp_data.extend(core.data_handler.data.GetTrace(n)) + model_data.extend(core.final_result[n]) + no_traces = core.data_handler.number_of_traces() + else: + for n in range(len(core.data_handler.features_data["stim_amp"])): + model_data.extend(core.final_result[n]) + no_traces = len(core.data_handler.features_data["stim_amp"]) + if core.option_handler.type[-1] != 'features' and core.option_handler.type[-1] != 'hippounit': + t = int(ceil(core.option_handler.input_length)) + else: + t = int(ceil(core.option_handler.run_controll_tstop)) + step = core.option_handler.run_controll_dt + axes.set_xticks([n for n in range(0, int((t * no_traces) / (step)), int((t * no_traces) / (step) / 5.0)) ]) + axes.set_xticklabels([str(n) for n in range(0, int(t * no_traces), int((t * no_traces) / 5))]) - - axes.set_xlabel("time [ms]") - if core.option_handler.type[-1]!= 'features': - _type = core.data_handler.data.type - else: - _type = "Voltage" if core.option_handler.run_controll_record =="v" else "Current" if core.option_handler.run_controll_record == "c" else "" - axes.set_ylabel(_type + " [" + core.option_handler.input_scale + "]") - if core.option_handler.type[-1]!= 'features': - axes.plot(list(range(0, len(exp_data))), exp_data) - axes.plot(list(range(0, len(model_data))), model_data, 'r') - axes.legend(["target", "model"]) - else: - axes.plot(list(range(0, len(model_data))), model_data, 'r') - axes.legend(["model"]) - fig.savefig("result_trace.png", dpi=None, facecolor='w', edgecolor='w', - orientation='portrait', papertype=None, format=None, - transparent=False, bbox_inches=None, pad_inches=0.1) - fig.savefig("result_trace.eps", dpi=None, facecolor='w', edgecolor='w') - fig.savefig("result_trace.svg", dpi=None, facecolor='w', edgecolor='w') + + axes.set_xlabel("time [ms]") + if core.option_handler.type[-1]!= 'features' and core.option_handler.type[-1] != 'hippounit': + _type = core.data_handler.data.type + else: + _type = "Voltage" if core.option_handler.run_controll_record =="v" else "Current" if core.option_handler.run_controll_record == "c" else "" + axes.set_ylabel(_type + " [" + core.option_handler.input_scale + "]") + if core.option_handler.type[-1]!= 'features' and core.option_handler.type[-1] != 'hippounit': + axes.plot(list(range(0, len(exp_data))), exp_data) + axes.plot(list(range(0, len(model_data))), model_data, 'r') + axes.legend(["target", "model"]) + else: + axes.plot(list(range(0, len(model_data))), model_data, 'r') + axes.legend(["model"]) + fig.savefig("result_trace.png", dpi=None, facecolor='w', edgecolor='w', + orientation='portrait', papertype=None, format=None, + transparent=False, bbox_inches=None, pad_inches=0.1) + fig.savefig("result_trace.eps", dpi=None, facecolor='w', edgecolor='w') + fig.savefig("result_trace.svg", dpi=None, facecolor='w', edgecolor='w') diff --git a/optimizer/fitnessFunctions.py b/optimizer/fitnessFunctions.py index 70402a44..7f22d4fc 100755 --- a/optimizer/fitnessFunctions.py +++ b/optimizer/fitnessFunctions.py @@ -14,6 +14,9 @@ import random import threading import matplotlib.pyplot as plt +import json +import collections +import traceback try: import copyreg @@ -92,6 +95,16 @@ def __init(self, start, start_val, peak, peak_val, stop, stop_val, spike): spike_frame.__init__(self, start, start_val, peak, peak_val, stop, stop_val) self.s = spike#vector, with the spike in it + +class fF_Factory: + @classmethod + def create(cls, reader_object, option_object): + if option_object.type[-1] == "hippounit": + return fF_HippoUnit(reader_object, option_object) + else: + return fF(reader_object, option_object) + + class fF(object): """ Class encapsulating the implemented error functions. @@ -125,10 +138,6 @@ def __init__(self, reader_object, option_object): "AHP depth": self.AHP_depth, "AP width": self.AP_width, "Derivative difference" : self.calc_grad_dif} - - - - def setParameters(self, section, params): """ @@ -156,9 +165,6 @@ def setParameters(self, section, params): #cal the user def.ed function usr_fun(self, params) - - - def modelRunner(self, candidates, act_trace_idx): """ Prepares the model for the simulation, runs the simulation and records the appropriate variable. @@ -1051,7 +1057,7 @@ def getErrorComponents(self, index_of_trace, model_output): add_data = None args = {} args["add_data"] = add_data - if (self.option.type[-1]!='features'): + if self.option.type[-1]!='features' and self.option.type[-1]!='hippounit': for f, w in zip(features, weigths): fit_list.append([w, f, (f(model_output, self.reader.data.GetTrace(index_of_trace), args))]) else: @@ -1157,3 +1163,115 @@ def multi_objective_fitness(self, candidates, args={}, delete_model=True): del self.model return self.fitnes + +class fF_HippoUnit(fF): + def __init__(self, reader_object, option_object): + super().__init__(reader_object, option_object) + self.model_trace = [] + self.is_figures_saved = False + self.model=modelHandler.modelHandlerHippounit(self.option) + self.tests_selected = self.model.settings["model"]["tests"] + self.tests_fitness = {} + + def select_test(self, test_name): + from hippounit import tests + + if test_name == "SomaticFeaturesTest": + with open(self.model.settings["tests"]["SomaticFeaturesTest"]["target_data_path"], "r") as target_data_file: + observation = json.load(target_data_file) + with open(self.model.settings["tests"]["SomaticFeaturesTest"]["stimuli_file_path"], "r") as stimuli_file: + config = json.load(stimuli_file) + return tests.SomaticFeaturesTest(observation=observation, config=config, force_run=True, show_plot=False, + save_all=self.is_figures_saved, base_directory=self.model.output_directory, + serialized=True) + elif test_name == "DepolarizationBlockTest": + with open(self.model.settings["tests"]["DepolarizationBlockTest"]["target_data_path"], + "r") as target_data_file: + observation = json.load(target_data_file) + return tests.DepolarizationBlockTest(observation=observation, force_run=True, + show_plot=False, save_all=self.is_figures_saved, + base_directory=self.model.output_directory, serialized=True) + elif test_name == "BackpropagatingAPTest": + with open(self.model.settings["tests"]["BackpropagatingAPTest"]["target_data_path"], "r") as f: + observation = json.load(f, object_pairs_hook=collections.OrderedDict) + with open(self.model.settings["tests"]["BackpropagatingAPTest"]["stimuli_file_path"], "r") as f: + config = json.load(f, object_pairs_hook=collections.OrderedDict) + return tests.BackpropagatingAPTest(config=config, observation=observation, force_run=True, + force_run_FindCurrentStim=True, show_plot=False, + save_all=self.is_figures_saved, base_directory=self.model.output_directory, + serialized=True) + elif test_name == "PSPAttenuationTest": + with open(self.model.settings["tests"]["PSPAttenuationTest"]["target_data_path"], "r") as f: + observation = json.load(f, object_pairs_hook=collections.OrderedDict) + with open(self.model.settings["tests"]["PSPAttenuationTest"]["stimuli_file_path"], "r") as f: + config = json.load(f, object_pairs_hook=collections.OrderedDict) + num_of_dend_locations = self.model.settings["tests"]["PSPAttenuationTest"]["num_of_dend_locations"] + #trunk_origin = self.model.settings["tests"]["PSPAttenuationTest"]["trunk_origin"] + self.model.model.SecList = self.model.model.TrunkSecList_name + return tests.PSPAttenuationTest(config=config, observation=observation, num_of_dend_locations=num_of_dend_locations, + force_run=True, show_plot=False, save_all=self.is_figures_saved, + base_directory=self.model.output_directory, serialized=True) + elif test_name == "ObliqueIntegrationTest": + with open(self.model.settings["tests"]["ObliqueIntegrationTest"]["target_data_path"], "r") as f: + observation = json.load(f, object_pairs_hook=collections.OrderedDict) + return tests.ObliqueIntegrationTest(observation=observation, save_all=self.is_figures_saved, + force_run_synapse=True, force_run_bin_search=True, show_plot=False, + base_directory=self.model.output_directory, serialized=True) + + def single_objective_fitness(self, candidates, args={}, delete_model=True): + os.chdir(self.option.base_dir) + + candidate_renormalized = self.ReNormalize(candidates[0]) + self.model.model.set_candidate(candidate_renormalized) + + self.fitnes = [] + error = 0 + for idx, test_name in enumerate(self.tests_selected): + test = self.select_test(test_name) + test.specify_data_set = self.model.settings["model"]["dataset"] + try: + score = self.model.run(test) + error += self.option.weights[idx] * score + self.tests_fitness[test_name] = score + except Exception as e: + print('Model: ' + self.model.model.name + ' could not be run') + traceback.print_stack() + + self.fitnes.append(error) + return self.fitnes + + def multi_objective_fitness(self, candidates, args={}, delete_model=True): + os.chdir(self.option.base_dir) + + candidate_renormalized = self.ReNormalize(candidates[0]) + self.model.model.set_candidate(candidate_renormalized) + + self.fitnes = [] + temp_fit = [] + + tests_selected = self.model.settings["model"]["tests"] + for test_name in tests_selected: + test = self.select_test(test_name) + test.specify_data_set = self.model.settings["model"]["dataset"] + + print("Running {test} on {model}".format(test=test, model=self.model.model.name)) + try: + error = self.model.run(test) + temp_fit.append(error) + self.model_trace = self.model.record + except Exception as e: + print('Model: ' + self.model.model.name + ' could not be run') + print(e) + traceback.print_stack() + + self.fitnes.append(temp_fit) + return self.fitnes + + def getTestErrorComponents(self): + fit_list = [] + for i in range(len(self.tests_selected)): + f = self.tests_selected[i] + w = self.option.weights[i] + e = self.tests_fitness[f] + fit_list.append([w, f, e]) + return fit_list diff --git a/optimizer/modelHandler.py b/optimizer/modelHandler.py index 81e0b63f..b097d573 100755 --- a/optimizer/modelHandler.py +++ b/optimizer/modelHandler.py @@ -4,6 +4,8 @@ import optimizerHandler import importlib import neuron +import json +import collections first_hoc_cls = neuron.h.__class__ first_hoc_dict = neuron.h.__dict__ @@ -441,3 +443,62 @@ def Recordings(self,vector): tr.Convert(vector) return tr.data # comment: pass the hoc vector to Convert, not the hoc_object + + +class modelHandlerHippounit: + def __init__(self, option_handler): + from hippounit.utils import ModelLoaderNeuroptimus + + self.option = option_handler + self.base_directory = self.option.base_dir + os.chdir(self.base_directory) + + self.model_path = self.option.model_path + #self.number_of_params = None + self.record = [] + #self.spike_times = None + + # load hippounit settings + self.settings = {} + self.load_settings() + + # create model loader + self.output_directory = self.settings["model"]["output_dir"] + mod_files_path = self.settings["model"]["mod_files_path"] + user_function = self.option.u_fun_string + self.model = ModelLoaderNeuroptimus(mod_files_path=mod_files_path, user_function_string=user_function) + self.set_model_parameters() + + # load observations and stimuli json + self.observation = {} + self.config = {} + #self.load_target_and_stimuli() + + def load_settings(self): + with open(self.option.hippounit_settings_path) as settings_file: + self.settings = json.load(settings_file) + + def set_model_parameters(self): + self.model.name = self.settings["model"]["name"] + self.model.hocpath = self.model_path + self.model.template_name = self.settings["model"]["template_name"] + self.model.SomaSecList_name = self.settings["model"]["SomaSecList_name"] + self.model.soma = self.settings["model"]["soma"] + self.model.v_init = self.settings["model"]["v_init"] + self.model.celsius = self.settings["model"]["celsius"] + self.model.TrunkSecList_name = self.settings["model"]["TrunkSecList_name"] + self.model.ObliqueSecList_name = self.settings["model"]["ObliqueSecList_name"] + self.model.cvode_active = False + + def load_target_and_stimuli(self): + target_data_path = self.settings["model"]["target_data_path"] + stimuli_file_path = self.settings["model"]["stimuli_file_path"] + with open(target_data_path) as target_file: + self.observation = json.load(target_file, object_pairs_hook=collections.OrderedDict) + with open(stimuli_file_path) as stimuli_file: + self.config = json.load(stimuli_file, object_pairs_hook=collections.OrderedDict) + + def run(self, test): + score = test.judge(self.model) + score.summarize() + return score.norm_score diff --git a/optimizer/optimizerHandler.py b/optimizer/optimizerHandler.py index ae61cfbe..82b80859 100755 --- a/optimizer/optimizerHandler.py +++ b/optimizer/optimizerHandler.py @@ -1,4 +1,4 @@ -from fitnessFunctions import fF,frange +from fitnessFunctions import fF_Factory,frange from optionHandler import optionHandler import sys import logging @@ -109,7 +109,7 @@ class SINGLERUN(): An abstract base class to implement a single evaluation process. """ def __init__(self, reader_obj, option_obj): - self.fit_obj = fF(reader_obj, option_obj) + self.fit_obj = fF_Factory.create(reader_obj, option_obj) self.SetFFun(option_obj) self.directory = option_obj.base_dir self.num_params = option_obj.num_params @@ -137,18 +137,20 @@ class baseOptimizer(): An abstract base class to implement the base of an optimization process. """ def __init__(self, reader_obj, option_obj): - self.fit_obj = fF(reader_obj, option_obj) + self.fit_obj = fF_Factory.create(reader_obj, option_obj) self.SetFFun(option_obj) self.rand = random self.seed = int(option_obj.seed) self.rand.seed(self.seed) self.directory = option_obj.base_dir self.num_params = option_obj.num_params - if option_obj.type[-1]!= "features": + if option_obj.type[-1]=="hippounit": + self.number_of_traces = None + elif option_obj.type[-1]!= "features": self.number_of_traces = reader_obj.number_of_traces() else: self.number_of_traces = len(reader_obj.features_data["stim_amp"]) - self.num_obj = self.num_params*int(self.number_of_traces) + #self.num_obj = self.num_params*int(self.number_of_traces) self.boundaries = option_obj.boundaries self.algo_params = copy.copy(option_obj.algorithm_parameters) @@ -164,7 +166,7 @@ def SetFFun(self,option_obj): except KeyError: sys.exit("Unknown fitness function!") - if option_obj.type[-1]!= 'features': + if option_obj.type[-1]!= 'features' and option_obj.type[-1]!='hippounit': try: option_obj.feats = [self.fit_obj.calc_dict[x] for x in option_obj.feats] except KeyError: @@ -275,7 +277,9 @@ def __init__(self, reader_obj, option_obj): self.pg.set_global_rng_seed(seed = self.seed) self.boundaries = [[0]*len(option_obj.boundaries[0]),[1]*len(option_obj.boundaries[1])] self.base_dir = option_obj.base_dir - if self.option_obj.type[-1]!="features": + if self.option_obj.type[-1]=="hippounit": + self.number_of_traces=1 # TODO: this is fake + elif self.option_obj.type[-1]!="features": self.number_of_traces=reader_obj.number_of_traces() else: self.number_of_traces=len(reader_obj.features_data["stim_amp"]) @@ -814,7 +818,7 @@ class grid(baseOptimizer): """ def __init__(self,reader_obj,option_obj,resolution): - self.fit_obj=fF(reader_obj,option_obj) + self.fit_obj=fF_Factory.create(reader_obj, option_obj) self.SetFFun(option_obj) self.num_params=option_obj.num_params self.num_points_per_dim=resolution diff --git a/optimizer/optionHandler.py b/optimizer/optionHandler.py index 22090148..eb8d47f8 100755 --- a/optimizer/optionHandler.py +++ b/optimizer/optionHandler.py @@ -139,17 +139,18 @@ def __init__(self): self.feats=[] self.feat_str=[] self.weights=[] + self.hippounit_settings_path=None + post=dir(self) self.class_content=list(OrderedSet(post)-OrderedSet(prev)) self.algorithm_parameters_dict=json.load(open(os.path.dirname(os.path.abspath(__file__))+"/algorithm_parameters.json", "r")) - def CreateDictForJson(self,f_mapper): json_dict={} for m in self.class_content: if m=="feats": - if self.type[-1]!='features': + if self.type[-1]!='features' and self.type[-1]!='hippounit': json_dict[m]=[f_mapper[x.__name__] for x in self.__getattribute__(m)] else: json_dict[m]=self.feats @@ -158,7 +159,6 @@ def CreateDictForJson(self,f_mapper): return {"selectable_algorithms":self.algorithm_parameters_dict,"attributes":json_dict} - def ReadJson(self,settings): for key, value in settings.items(): self.__setattr__(key,value) @@ -170,8 +170,6 @@ def ReadJson(self,settings): self.algorithm_parameters=self.algorithm_parameters_dict[self.algorithm_name] self.algorithm_parameters.update(list(self.current_algorithm.values())[0]) print(self.algorithm_parameters) - - # returns the current settings of the current working directory (referred as base in modelHandler, used in traceReader ) def GetFileOption(self): diff --git a/optimizer/traceHandler.py b/optimizer/traceHandler.py index f75b569f..8317efdc 100755 --- a/optimizer/traceHandler.py +++ b/optimizer/traceHandler.py @@ -448,8 +448,6 @@ def abstractDataReader(self,path): sys.exit("Can't open data file at " + my_file + " ! Exiting...") return data_dict - - # class to write data to file #it receives a trace object and other settings #comment: comments to the file