From c630432c5a8cc6c9fbe04ee78ca00290f134beef Mon Sep 17 00:00:00 2001 From: Anna Wirbel Date: Fri, 5 Jan 2024 10:35:27 +0100 Subject: [PATCH] add contours plot add runscripts add function to create local cfg files with overrides add collection name to override section update docs move writing of local cfgs to folder add option to read config from dir --- avaframe/ana5Utils/distanceTimeAnalysis.py | 6 +- avaframe/avaframeCfg.ini | 3 +- avaframe/com1DFA/com1DFA.py | 26 +- avaframe/com1DFA/com1DFATools.py | 3 +- avaframe/in3Utils/cfgHandling.py | 8 + avaframe/in3Utils/cfgUtils.py | 608 +++++++++--------- avaframe/out3Plot/plotUtils.py | 2 +- avaframe/runCom1DFA.py | 75 ++- avaframe/runScripts/runPlotContoursFromAsc.py | 2 +- docs/configuration.rst | 10 + 10 files changed, 399 insertions(+), 344 deletions(-) diff --git a/avaframe/ana5Utils/distanceTimeAnalysis.py b/avaframe/ana5Utils/distanceTimeAnalysis.py index 638c227b1..c5afe79eb 100644 --- a/avaframe/ana5Utils/distanceTimeAnalysis.py +++ b/avaframe/ana5Utils/distanceTimeAnalysis.py @@ -559,7 +559,7 @@ def extractFrontAndMeanValuesTT(cfgRangeTime, flowF, demHeader, mtiInfo): return mtiInfo -def initializeRangeTime(modName, cfg, dem, simHash): +def initializeRangeTime(modName, cfg, dem, simHash, configDir): """ initialize generation of range-time diagram for visualizing simulation data Parameters @@ -572,6 +572,8 @@ def initializeRangeTime(modName, cfg, dem, simHash): dictionary with DEM header and data simHash: str unique simulation ID + configDir: str or pathlib path + path to configuration directory - optional if not provided has to be empty string Returns -------- @@ -583,7 +585,7 @@ def initializeRangeTime(modName, cfg, dem, simHash): """ # fetch configuration and add info - cfgRangeTime = cfgUtils.getModuleConfig(modName) + cfgRangeTime = cfgUtils.getModuleConfig(modName, configDir) cfgRangeTime['GENERAL']['tEnd'] = cfg['GENERAL']['tEnd'] cfgRangeTime['GENERAL']['avalancheDir'] = cfg['GENERAL']['avalancheDir'] diff --git a/avaframe/avaframeCfg.ini b/avaframe/avaframeCfg.ini index 1e43f3fc3..5689d5526 100644 --- a/avaframe/avaframeCfg.ini +++ b/avaframe/avaframeCfg.ini @@ -5,7 +5,8 @@ [MAIN] # Path to avalanche directory avalancheDir = data/avaParabola - +# OPTIONAL Path to configuration file directory +configurationDir = # number of CPU cores to use for the computation of com1DFA # possible values are: # - auto -> takes up to CPUPercent (see below) % of the available CPU cores diff --git a/avaframe/com1DFA/com1DFA.py b/avaframe/com1DFA/com1DFA.py index f2e8d58c6..5ca3e636a 100644 --- a/avaframe/com1DFA/com1DFA.py +++ b/avaframe/com1DFA/com1DFA.py @@ -93,7 +93,7 @@ def com1DFAPreprocess(cfgMain, typeCfgInfo, cfgInfo): # read initial configuration if typeCfgInfo in ["cfgFromFile", "cfgFromDefault"]: - cfgStart = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgInfo, toPrint=False) + cfgStart = cfgUtils.getModuleConfig(com1DFA, cfgMain['MAIN']['configurationDir'], fileOverride=cfgInfo, toPrint=False) elif typeCfgInfo == "cfgFromObject": cfgStart = cfgInfo @@ -167,7 +167,7 @@ def com1DFAMain(cfgMain, cfgInfo=""): nCPU = cfgUtils.getNumberOfProcesses(cfgMain, len(simDict)) # Supply compute task with inputs - com1DFACoreTaskWithInput = partial(com1DFACoreTask, simDict, inputSimFiles, avalancheDir, outDir) + com1DFACoreTaskWithInput = partial(com1DFACoreTask, simDict, inputSimFiles, cfgMain, outDir) # Create parallel pool and run # with multiprocessing.Pool(processes=nCPU) as pool: @@ -200,11 +200,13 @@ def com1DFAMain(cfgMain, cfgInfo=""): return 0, {}, [], "" -def com1DFACoreTask(simDict, inputSimFiles, avalancheDir, outDir, cuSim): +def com1DFACoreTask(simDict, inputSimFiles, cfgMain, outDir, cuSim): """This is a subdivision of com1DFAMain to allow for parallel execution. Please read this in the context of the com1DFAMain function. """ + avalancheDir = cfgMain['MAIN']['avalancheDir'] + simDF = pd.DataFrame() tCPUDF = pd.DataFrame() @@ -232,7 +234,7 @@ def com1DFACoreTask(simDict, inputSimFiles, avalancheDir, outDir, cuSim): cfgFinal, tCPU, particlesList, - ) = com1DFA.com1DFACore(cfg, avalancheDir, cuSim, inputSimFiles, outDir, simHash=simHash) + ) = com1DFA.com1DFACore(cfg, cfgMain, cuSim, inputSimFiles, outDir, simHash=simHash) simDF.at[simHash, "nPart"] = str(int(particlesList[0]["nPart"])) @@ -323,7 +325,7 @@ def com1DFAPostprocess(simDF, tCPUDF, simDFExisting, cfgMain, dem, reportDictLis return dem, plotDict, reportDictList, simDFNew -def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): +def com1DFACore(cfg, cfgMain, cuSimName, inputSimFiles, outDir, simHash=""): """Run main com1DFA model This will compute a dense flow avalanche with the settings specified in cfg and the name cuSimName @@ -332,12 +334,12 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): ---------- cfg : configparser object configuration object for simulation to be performed + cfgMain: configparser object + main configuration of AvaFrame used here: avalancheDir, configurationDir cuSimName: str name of simulation inputSimFiles: dict dictionary with input files, release scenario chosen according to inputSimFiles['releaseScenario'] - avaDir : str or pathlib object - path to avalanche directory outDir: str or pathlib object path to Outputs simHash: str @@ -359,6 +361,8 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): list of particle dictionaries for all saving time steps """ + avaDir = cfgMain['MAIN']['avalancheDir'] + # select release area input data according to chosen release scenario inputSimFiles = gI.selectReleaseFile(inputSimFiles, cfg["INPUT"]["releaseScenario"]) @@ -388,7 +392,7 @@ def com1DFACore(cfg, avaDir, cuSimName, inputSimFiles, outDir, simHash=""): # ------------------------ # Start time step computation Tsave, particlesList, fieldsList, infoDict = DFAIterate( - cfg, particles, fields, dem, inputSimLines, simHash=simHash + cfg, particles, fields, dem, inputSimLines, cfgMain['MAIN']['configurationDir'], simHash=simHash ) # write mass balance to File @@ -1657,7 +1661,7 @@ def initializeResistance(cfg, dem, simTypeActual, resLine, reportAreaInfo, thres return cResRaster, detRaster, reportAreaInfo -def DFAIterate(cfg, particles, fields, dem, inputSimLines, simHash=""): +def DFAIterate(cfg, particles, fields, dem, inputSimLines, configDir, simHash=""): """Perform time loop for DFA simulation Save results at desired intervals @@ -1676,6 +1680,8 @@ def DFAIterate(cfg, particles, fields, dem, inputSimLines, simHash=""): dictionary with dem information inputSimLines : dict dictionary with input data dictionaries (releaseLine, entLine, ...) + configDir: str or pathlib Path + path to configuration directory - optional if not provided has to be empty string Returns ------- @@ -1762,7 +1768,7 @@ def DFAIterate(cfg, particles, fields, dem, inputSimLines, simHash=""): # check if range-time diagram should be performed, if yes - initialize if cfg["VISUALISATION"].getboolean("createRangeTimeDiagram"): demRT = dtAna.setDemOrigin(dem) - mtiInfo, dtRangeTime, cfgRangeTime = dtAna.initializeRangeTime(dtAna, cfg, demRT, simHash) + mtiInfo, dtRangeTime, cfgRangeTime = dtAna.initializeRangeTime(dtAna, cfg, demRT, simHash, configDir) # fetch initial time step too mtiInfo, dtRangeTime = dtAna.fetchRangeTimeInfo( cfgRangeTime, cfg, dtRangeTime, t, demRT["header"], fields, mtiInfo diff --git a/avaframe/com1DFA/com1DFATools.py b/avaframe/com1DFA/com1DFATools.py index a80d41296..c27aa91b0 100644 --- a/avaframe/com1DFA/com1DFATools.py +++ b/avaframe/com1DFA/com1DFATools.py @@ -283,7 +283,8 @@ def createSimDictFromCfgs(cfgMain, cfgPath): # loop over all cfgFiles and create simDict for index, cfgFile in enumerate(cfgFilesAll): # read configuration - cfgFromFile = cfgUtils.getModuleConfig(com1DFA, fileOverride=cfgFile, toPrint=False) + # configDir is set to '' because fileOverride is provided + cfgFromFile = cfgUtils.getModuleConfig(com1DFA, '', fileOverride=cfgFile, toPrint=False) # create dictionary with one key for each simulation that shall be performed # NOTE: sims that are added don't need to be added to the simNameExisting list as diff --git a/avaframe/in3Utils/cfgHandling.py b/avaframe/in3Utils/cfgHandling.py index 9446ddb76..6f82e1d8e 100644 --- a/avaframe/in3Utils/cfgHandling.py +++ b/avaframe/in3Utils/cfgHandling.py @@ -540,6 +540,11 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): optional - path to directory to store local_ cfg ini file to if not provided - local_ cfg ini file is saved to avalanche directory + Returns + -------- + locFilePath: pathlib Path + path to directory where local configuration files are written to derived from override sections + """ # if a path is provided - save local cfg ini file there @@ -576,6 +581,7 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): cfgModule = cfgUtils.getModuleConfig( cfgNamePath, + '', fileOverride="", modInfo=False, toPrint=False, @@ -606,6 +612,8 @@ def rewriteLocalCfgs(cfgFull, avalancheDir, localCfgPath=''): log.info("%s CONFIGURATION wrote to %s" % (cfgName, str(cfgF))) + return locFilePath + def _removeCfgItemsNotInOverride(cfgModule, overrideKeys): """ remove options of cfgModule if not part of overrideKeys diff --git a/avaframe/in3Utils/cfgUtils.py b/avaframe/in3Utils/cfgUtils.py index 5a98b5e52..db11a4b58 100644 --- a/avaframe/in3Utils/cfgUtils.py +++ b/avaframe/in3Utils/cfgUtils.py @@ -1,7 +1,7 @@ -''' +""" Utilities for handling configuration files -''' +""" import configparser import logging @@ -27,25 +27,25 @@ log = logging.getLogger(__name__) -def getGeneralConfig(nameFile=''): - ''' Returns the general configuration for avaframe +def getGeneralConfig(nameFile=""): + """Returns the general configuration for avaframe returns a configParser object Parameters ---------- nameFile: pathlib path optional full path to file, if empty use avaframeCfg from folder one level up - ''' + """ # get path of module modPath = pathlib.Path(avaf.__file__).resolve().parent if isinstance(nameFile, pathlib.Path): - localFile = nameFile.parents[0] / ('local_' + nameFile.name) + localFile = nameFile.parents[0] / ("local_" + nameFile.name) defaultFile = nameFile else: - localFile = modPath / 'local_avaframeCfg.ini' - defaultFile = modPath / 'avaframeCfg.ini' + localFile = modPath / "local_avaframeCfg.ini" + defaultFile = modPath / "avaframeCfg.ini" if localFile.is_file(): iniFile = localFile @@ -55,16 +55,16 @@ def getGeneralConfig(nameFile=''): iniFile = defaultFile compare = False else: - raise FileNotFoundError('None of the provided cfg files exist ') + raise FileNotFoundError("None of the provided cfg files exist ") # Finally read it - cfg, _ = readCompareConfig(iniFile, 'General', compare) + cfg, _ = readCompareConfig(iniFile, "General", compare) return cfg -def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDefault=False): - ''' Returns the configuration for a given module +def getModuleConfig(module, configDir, fileOverride="", modInfo=False, toPrint=True, onlyDefault=False): + """Returns the configuration for a given module returns a configParser object module object: module : the calling function provides the already imported @@ -76,6 +76,9 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe leads to getModuleConfig(c2) OR: pathlib Path to module (python file) + configDir: pathlib Path or str + directory to local configuration files - if not provided has to be empty str! + Str: fileOverride : allows for a completely different file location. However note: missing values from the default cfg will always be added! @@ -87,10 +90,10 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe Order is as follows: fileOverride -> local_MODULECfg.ini -> MODULECfg.ini - ''' + """ if isinstance(onlyDefault, bool) == False: - message = 'OnlyDefault parameter is not a boolean but %s' % type(onlyDefault) + message = "OnlyDefault parameter is not a boolean but %s" % type(onlyDefault) log.error(message) raise TypeError(message) @@ -101,11 +104,22 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe else: modPath, modName = getModPathName(module) - localFile = modPath / ('local_'+modName+'Cfg.ini') - defaultFile = modPath / (modName+'Cfg.ini') + if configDir != "": + if pathlib.Path(configDir).is_dir() is False: + message = "Provided configurationDir: %s is not a directory!" % str(configDir) + log.error(message) + raise NotADirectoryError(message) + else: + directoryFile = pathlib.Path(configDir, (("local_" + modName + "Cfg.ini"))) + else: + directoryFile = None - log.debug('localFile: %s', localFile) - log.debug('defaultFile: %s', defaultFile) + localFile = modPath / ("local_" + modName + "Cfg.ini") + defaultFile = modPath / (modName + "Cfg.ini") + + log.debug("localFile: %s", localFile) + log.debug("defaultFile: %s", defaultFile) + log.debug("directoryFile: %s", directoryFile) # Decide which one to take if fileOverride: @@ -114,9 +128,10 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe iniFile = [defaultFile, fileOverride] compare = True else: - raise FileNotFoundError('Provided fileOverride does not exist: ' + - str(fileOverride)) - + raise FileNotFoundError("Provided fileOverride does not exist: " + str(fileOverride)) + elif (directoryFile is not None) and directoryFile.is_file(): + iniFile = [defaultFile, directoryFile] + compare = True elif localFile.is_file() and not onlyDefault: iniFile = localFile iniFile = [defaultFile, localFile] @@ -125,7 +140,7 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe iniFile = defaultFile compare = False else: - raise FileNotFoundError('None of the provided cfg files exist ') + raise FileNotFoundError("None of the provided cfg files exist ") # Finally read it cfg, modDict = readCompareConfig(iniFile, modName, compare, toPrint) @@ -137,7 +152,7 @@ def getModuleConfig(module, fileOverride='', modInfo=False, toPrint=True, onlyDe def getDefaultModuleConfig(module, toPrint=True): - ''' Returns the default configuration for a given module + """Returns the default configuration for a given module returns a configParser object module object: module : the calling function provides the already imported @@ -148,15 +163,14 @@ def getDefaultModuleConfig(module, toPrint=True): from avaframe.com2AB import com2AB as c2 leads to getModuleConfig(c2) - ''' + """ # get path to the module and its name modPath, modName = getModPathName(module) + defaultFile = modPath / (modName + "Cfg.ini") - defaultFile = modPath / (modName+'Cfg.ini') - - log.info('Getting the default config for %s', modName) - log.debug('defaultFile: %s', defaultFile) + log.info("Getting the default config for %s", modName) + log.debug("defaultFile: %s", defaultFile) # Finally read it cfg, _ = readCompareConfig(defaultFile, modName, compare=False, toPrint=toPrint) @@ -165,7 +179,7 @@ def getDefaultModuleConfig(module, toPrint=True): def readCompareConfig(iniFile, modName, compare, toPrint=True): - ''' Read and optionally compare configuration files (if a local and default are both provided) + """Read and optionally compare configuration files (if a local and default are both provided) and inform user of the eventual differences. Take the default as reference. Parameters @@ -184,10 +198,10 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): contains combined config modDict: dict dictionary containing only differences from default - ''' + """ if compare: - log.info('Reading config from: %s and %s' % (iniFile[0], iniFile[1])) + log.info("Reading config from: %s and %s" % (iniFile[0], iniFile[1])) # initialize configparser object to read defCfg = configparser.ConfigParser() defCfg.optionxform = str @@ -197,12 +211,12 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): defCfg.read(iniFile[0]) locCfg.read(iniFile[1]) - log.debug('Writing cfg for: %s', modName) + log.debug("Writing cfg for: %s", modName) # compare to default config and get modification dictionary and config modDict, modCfg = compareTwoConfigs(defCfg, locCfg, toPrint=toPrint) else: - log.info('Reading config from: %s', iniFile) + log.info("Reading config from: %s", iniFile) # initialize our final configparser object modCfg = configparser.ConfigParser() modCfg.optionxform = str @@ -217,58 +231,58 @@ def readCompareConfig(iniFile, modName, compare, toPrint=True): def _splitDeepDiffValuesChangedItem(inKey, inVal): - """ splits one item of a deepdiff result into section, key, old value, new value - - Parameters - ----------- - inputKey: str - key of a deepdiff changed_values item - inputValue: dict - value of a deepdiff changed_values item - - Returns - -------- - section: str - section name of changed item - key: str - key name of changed item - oldVal: str - old value - newVal: str - new value + """splits one item of a deepdiff result into section, key, old value, new value + + Parameters + ----------- + inputKey: str + key of a deepdiff changed_values item + inputValue: dict + value of a deepdiff changed_values item + + Returns + -------- + section: str + section name of changed item + key: str + key name of changed item + oldVal: str + old value + newVal: str + new value """ splitKey = re.findall(r"\['?([A-Za-z0-9_]+)'?\]", inKey) section = splitKey[0] key = splitKey[1] - return section, key, inVal['old_value'], inVal['new_value'] + return section, key, inVal["old_value"], inVal["new_value"] def compareTwoConfigs(defCfg, locCfg, toPrint=False): - """ compare locCfg to defCfg and return a cfg object and modification dict - Values are merged from locCfg to defCfg: - - parameters already in defCfg get the value from locCfg - - additional values in locCfg get added in the resulting Cfg - - Parameters - ----------- - defCfg: configparser object - default configuration - locCfg: configuration object - configuration that is compared to defCfg - toPrint: bool - flag if config shall be printed to log - - Returns - -------- - modInfo: dict - dictionary containing only differences from default - cfg: configParser object - contains combined config + """compare locCfg to defCfg and return a cfg object and modification dict + Values are merged from locCfg to defCfg: + - parameters already in defCfg get the value from locCfg + - additional values in locCfg get added in the resulting Cfg + + Parameters + ----------- + defCfg: configparser object + default configuration + locCfg: configuration object + configuration that is compared to defCfg + toPrint: bool + flag if config shall be printed to log + + Returns + -------- + modInfo: dict + dictionary containing only differences from default + cfg: configParser object + contains combined config """ - log.info('Comparing two configs') + log.info("Comparing two configs") # initialize modInfo and printOutInfo modInfo = dict() @@ -301,12 +315,12 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): # If toPrint is set, print full configuration: if toPrint: - for line in pformat(modCfgD, sort_dicts=False).split('\n'): + for line in pformat(modCfgD, sort_dicts=False).split("\n"): log.info(line) # Generate modInfo dictionary for output - if 'values_changed' in cfgDiff: - for key, value in cfgDiff['values_changed'].items(): + if "values_changed" in cfgDiff: + for key, value in cfgDiff["values_changed"].items(): section, itemKey, defValue, locValue = _splitDeepDiffValuesChangedItem(key, value) if section not in modInfo: @@ -316,91 +330,91 @@ def compareTwoConfigs(defCfg, locCfg, toPrint=False): modInfo[section][itemKey] = modString # Log changes - log.info('COMPARING TO DEFAULT, THESE CHANGES HAPPENED:') - for line in cfgDiff.pretty().split('\n'): - log.info(line.replace('root','')) + log.info("COMPARING TO DEFAULT, THESE CHANGES HAPPENED:") + for line in cfgDiff.pretty().split("\n"): + log.info(line.replace("root", "")) return modInfo, modCfg -def writeCfgFile(avaDir, module, cfg, fileName='', filePath=''): - """ Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini - or optional to filePath and with fileName +def writeCfgFile(avaDir, module, cfg, fileName="", filePath=""): + """Save configuration used to text file in Outputs/moduleName/configurationFiles/modName.ini + or optional to filePath and with fileName - Parameters - ----------- - avaDir: str - path to avalanche directory - module: - module - cfg: configparser object - configuration settings - fileName: str - name of saved configuration file - optional - filePath: str or pathlib path - path where file should be saved to except file name - optional + Parameters + ----------- + avaDir: str + path to avalanche directory + module: + module + cfg: configparser object + configuration settings + fileName: str + name of saved configuration file - optional + filePath: str or pathlib path + path where file should be saved to except file name - optional """ # get filename of module name = pathlib.Path(module.__file__).name - modName = name.split('.')[0] + modName = name.split(".")[0] # set outputs - if filePath == '': - outDir = pathlib.Path(avaDir, 'Outputs', modName, 'configurationFiles') + if filePath == "": + outDir = pathlib.Path(avaDir, "Outputs", modName, "configurationFiles") fU.makeADir(outDir) else: if filePath.is_dir(): outDir = pathlib.Path(filePath) else: - message = '%s is not a valid location for saving cfg file' % str(filePath) + message = "%s is not a valid location for saving cfg file" % str(filePath) log.error(message) raise NotADirectoryError(message) # set path to file - if fileName == '': + if fileName == "": fileName = modName - pathToFile = pathlib.Path(outDir, '%s.ini' % (fileName)) + pathToFile = pathlib.Path(outDir, "%s.ini" % (fileName)) # write file - with open(pathToFile, 'w') as conf: + with open(pathToFile, "w") as conf: cfg.write(conf) return pathToFile -def readCfgFile(avaDir, module='', fileName=''): - """ Read configuration from ini file, if module is provided, module configuration is read from Ouputs, - if fileName is provided configuration is read from fileName +def readCfgFile(avaDir, module="", fileName=""): + """Read configuration from ini file, if module is provided, module configuration is read from Ouputs, + if fileName is provided configuration is read from fileName - Parameters - ----------- - avaDir: str - path to avalanche directory - module: - module - fileName: str - path to file that should be read - optional + Parameters + ----------- + avaDir: str + path to avalanche directory + module: + module + fileName: str + path to file that should be read - optional - Returns - -------- - cfg: configParser object - configuration that is from file + Returns + -------- + cfg: configParser object + configuration that is from file """ # define file that should be read - if fileName != '': + if fileName != "": inFile = fileName - elif module != '': + elif module != "": # get module name name = pathlib.Path(module.__file__).name - modName = name.split('.')[0] + modName = name.split(".")[0] # set input file - inFile = pathlib.Path(avaDir, 'Outputs', '%s_settings.ini' % (modName)) + inFile = pathlib.Path(avaDir, "Outputs", "%s_settings.ini" % (modName)) else: - log.error('Please provide either a module or a fileName to read configuration from file') + log.error("Please provide either a module or a fileName to read configuration from file") raise NameError # read configParser object from input file, case sensitive @@ -413,7 +427,7 @@ def readCfgFile(avaDir, module='', fileName=''): def cfgHash(cfg, typeDict=False): - """ UID hash of a config. Given a configParser object cfg, + """UID hash of a config. Given a configParser object cfg, or a dictionary - then typeDict=True, returns a uid hash Parameters @@ -445,7 +459,7 @@ def cfgHash(cfg, typeDict=False): def convertConfigParserToDict(cfg): - """ create dictionary from configparser object """ + """create dictionary from configparser object""" cfgDict = {} for section in cfg.sections(): @@ -457,7 +471,7 @@ def convertConfigParserToDict(cfg): def convertDictToConfigParser(cfgDict): - """ create configParser object from dict """ + """create configParser object from dict""" cfg = configparser.ConfigParser() cfg.optionxform = str @@ -468,7 +482,7 @@ def convertDictToConfigParser(cfgDict): def writeDictToJson(inDict, outFilePath): - """ write a dictionary to a json file """ + """write a dictionary to a json file""" jsonDict = json.dumps(inDict, sort_keys=True, ensure_ascii=True) f = open(outFilePath, "w") @@ -476,54 +490,54 @@ def writeDictToJson(inDict, outFilePath): f.close() -def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCSV=False, specDir=''): - """ Read configurations from all simulations configuration ini files from directory - - Parameters - ----------- - avaDir: str - path to avalanche directory - standardCfg: dict - standard configuration for module - option - writeCSV: bool - True if configuration dataFrame shall be written to csv file - specDir: str - path to a directory where simulation configuration files can be found - optional - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations +def createConfigurationInfo(avaDir, comModule="com1DFA", standardCfg="", writeCSV=False, specDir=""): + """Read configurations from all simulations configuration ini files from directory + + Parameters + ----------- + avaDir: str + path to avalanche directory + standardCfg: dict + standard configuration for module - option + writeCSV: bool + True if configuration dataFrame shall be written to csv file + specDir: str + path to a directory where simulation configuration files can be found - optional + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', comModule, 'configurationFiles') - configFiles = inDir.glob('*.ini') + inDir = pathlib.Path(avaDir, "Outputs", comModule, "configurationFiles") + configFiles = inDir.glob("*.ini") if not inDir.is_dir(): - message = 'configuration file directory not found: %s' % (inDir) + message = "configuration file directory not found: %s" % (inDir) log.error(message) raise NotADirectoryError(message) elif configFiles == []: - message = 'No configuration file found in: %s' % (inDir) + message = "No configuration file found in: %s" % (inDir) log.error(message) raise FileNotFoundError(message) # create confiparser object, convert to json object, write to dataFrame # append all dataFrames - simDF = '' + simDF = "" for cFile in configFiles: - if 'sourceConfiguration' not in str(cFile): + if "sourceConfiguration" not in str(cFile): simName = pathlib.Path(cFile).stem - if '_AF_' in simName: - nameParts = simName.split('_AF_') - infoParts = nameParts[1].split('_') + if "_AF_" in simName: + nameParts = simName.split("_AF_") + infoParts = nameParts[1].split("_") else: - nameParts = simName.split('_') + nameParts = simName.split("_") infoParts = nameParts[1:] simHash = infoParts[0] cfgObject = readCfgFile(avaDir, fileName=cFile) @@ -533,9 +547,9 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS simDF = convertDF2numerics(simDF) # add default configuration - if standardCfg != '': + if standardCfg != "": # read default configuration of this module - simDF = appendCgf2DF('current standard', 'current standard', standardCfg, simDF) + simDF = appendCgf2DF("current standard", "current standard", standardCfg, simDF) # if writeCSV, write dataFrame to csv file if writeCSV: @@ -545,31 +559,31 @@ def createConfigurationInfo(avaDir, comModule='com1DFA', standardCfg='', writeCS def appendCgf2DF(simHash, simName, cfgObject, simDF): - """ append simulation configuration to the simulation dataframe - only account for sections GENERAL and INPUT - - Parameters - ----------- - simHash: str - hash of the simulation to append - simName: str - name of the simulation - cfgObject: configParser - configuration coresponding to the simulation - simDF: pandas dataFrame - configuration dataframe - - Returns - -------- - simDF: pandas DataFrame - DFappended with the new simulation configuration + """append simulation configuration to the simulation dataframe + only account for sections GENERAL and INPUT + + Parameters + ----------- + simHash: str + hash of the simulation to append + simName: str + name of the simulation + cfgObject: configParser + configuration coresponding to the simulation + simDF: pandas dataFrame + configuration dataframe + + Returns + -------- + simDF: pandas DataFrame + DFappended with the new simulation configuration """ indexItem = [simHash] cfgDict = convertConfigParserToDict(cfgObject) - simItemDFGeneral = pd.DataFrame(data=cfgDict['GENERAL'], index=indexItem) - simItemDFInput = pd.DataFrame(data=cfgDict['INPUT'], index=indexItem) - if 'VISUALISATION' in cfgDict: - simItemDFVisualisation = pd.DataFrame(data=cfgDict['VISUALISATION'], index=indexItem) + simItemDFGeneral = pd.DataFrame(data=cfgDict["GENERAL"], index=indexItem) + simItemDFInput = pd.DataFrame(data=cfgDict["INPUT"], index=indexItem) + if "VISUALISATION" in cfgDict: + simItemDFVisualisation = pd.DataFrame(data=cfgDict["VISUALISATION"], index=indexItem) simItemDF = pd.concat([simItemDFGeneral, simItemDFInput, simItemDFVisualisation], axis=1) else: simItemDF = pd.concat([simItemDFGeneral, simItemDFInput], axis=1) @@ -582,21 +596,21 @@ def appendCgf2DF(simHash, simName, cfgObject, simDF): def appendTcpu2DF(simHash, tCPU, tCPUDF): - """ append Tcpu dictionary to the dataframe - - Parameters - ----------- - simHash: str - hash of the simulation corresponding to the tCPU dict to append - tCPU: dict - cpu time dict of the simulation - tCPUDF: pandas dataFrame - tCPU dataframe - - Returns - -------- - simDF: pandas DataFrame - DFappended with the new simulation configuration + """append Tcpu dictionary to the dataframe + + Parameters + ----------- + simHash: str + hash of the simulation corresponding to the tCPU dict to append + tCPU: dict + cpu time dict of the simulation + tCPUDF: pandas dataFrame + tCPU dataframe + + Returns + -------- + simDF: pandas DataFrame + DFappended with the new simulation configuration """ indexItem = [simHash] tCPUItemDF = pd.DataFrame(data=tCPU, index=indexItem) @@ -608,99 +622,99 @@ def appendTcpu2DF(simHash, tCPU, tCPUDF): def convertDF2numerics(simDF): - """ convert a string DF to a numerical one + """convert a string DF to a numerical one - Parameters - ----------- - simDF: pandas dataFrame - dataframe + Parameters + ----------- + simDF: pandas dataFrame + dataframe - Returns - -------- - simDF: pandas DataFrame + Returns + -------- + simDF: pandas DataFrame """ for name, values in simDF.items(): - simDFTest = simDF[name].str.replace('.', '', regex=False) + simDFTest = simDF[name].str.replace(".", "", regex=False) # allow for - sign too - simDFTest = simDFTest.replace('-', '', regex=False) + simDFTest = simDFTest.replace("-", "", regex=False) # check for str(np.nan) as these cannot be converted to numerics by pd.to_numeric # but as friction model parameters are set to nans this is required here - if simDFTest.str.match('nan').any(): + if simDFTest.str.match("nan").any(): simDF = setStrnanToNan(simDF, simDFTest, name) # also include columns where nan is in first row - so check for any row if simDFTest.str.isdigit().any() and (name != 'tSteps'): # problem here is that it finds even if not present in | although not in ini - simDFTest = simDF[name].str.replace('|', '§', regex=False) - if simDFTest.str.contains('§').any() == False: + simDFTest = simDF[name].str.replace("|", "§", regex=False) + if simDFTest.str.contains("§").any() == False: simDF[name] = pd.to_numeric(simDF[name]) - log.debug('Converted to numeric %s' % name) + log.debug("Converted to numeric %s" % name) else: - log.debug('Not converted to numeric: %s' % name) + log.debug("Not converted to numeric: %s" % name) return simDF def setStrnanToNan(simDF, simDFTest, name): - """ set pandas element to np.nan if it is a string nan - - Parameters - ----------- - simDF: pandas dataFrame - dataframe - simDFTest: pandas series - series of sim DF column named name - replaced "." with " " - name: str - name of pandas dataframe column - - Returns - -------- - simDF: pandas dataframe - updated pandas dataframe with np.nan values where string nan was + """set pandas element to np.nan if it is a string nan + + Parameters + ----------- + simDF: pandas dataFrame + dataframe + simDFTest: pandas series + series of sim DF column named name + replaced "." with " " + name: str + name of pandas dataframe column + + Returns + -------- + simDF: pandas dataframe + updated pandas dataframe with np.nan values where string nan was """ - nanIndex = simDFTest.str.match('nan', flags=re.IGNORECASE) + nanIndex = simDFTest.str.match("nan", flags=re.IGNORECASE) simIndex = simDF.index.values # loop over each row and use simDF.at to avoid copy vs view warning for index, nanInd in enumerate(nanIndex): if nanInd: simDF.at[simIndex[index], name] = np.nan - log.info('%s for index: %s set to numpy nan' % (name, index)) + log.info("%s for index: %s set to numpy nan" % (name, index)) return simDF -def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfigurations'): - """ Read allConfigurations.csv file as dataFrame from directory - - Parameters - ----------- - avaDir: str - path to avalanche directory - specDir: str - path to a directory where simulation configuration files can be found - optional - configCsvName: str - name of configuration csv file - - Returns - -------- - simDF: pandas DataFrame - DF with all the simulation configurations - simDFName: array - simName column of the dataframe +def readAllConfigurationInfo(avaDir, specDir="", configCsvName="allConfigurations"): + """Read allConfigurations.csv file as dataFrame from directory + + Parameters + ----------- + avaDir: str + path to avalanche directory + specDir: str + path to a directory where simulation configuration files can be found - optional + configCsvName: str + name of configuration csv file + + Returns + -------- + simDF: pandas DataFrame + DF with all the simulation configurations + simDFName: array + simName column of the dataframe """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') - configFiles = inDir / ('%s.csv' % configCsvName) + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") + configFiles = inDir / ("%s.csv" % configCsvName) if configFiles.is_file(): - with open(configFiles, 'rb') as file: + with open(configFiles, "rb") as file: simDF = pd.read_csv(file, index_col=0, keep_default_na=False) - simDFName = simDF['simName'].to_numpy() + simDFName = simDF["simName"].to_numpy() else: simDF = None simDFName = [] @@ -708,31 +722,31 @@ def readAllConfigurationInfo(avaDir, specDir='', configCsvName='allConfiguration return simDF, simDFName -def writeAllConfigurationInfo(avaDir, simDF, specDir='', csvName='allConfigurations.csv'): - """ Write cfg configuration to allConfigurations.csv - - Parameters - ----------- - avaDir: str - path to avalanche directory - simDF: pandas dataFrame - daaframe of the configuration - specDir: str - path to a directory where simulation configuration shal be saved - optional - csvName: str - name of csv file in which to save to - optional - - Returns - -------- - configFiles: pathlib Path - path where the configuration dataframe was saved +def writeAllConfigurationInfo(avaDir, simDF, specDir="", csvName="allConfigurations.csv"): + """Write cfg configuration to allConfigurations.csv + + Parameters + ----------- + avaDir: str + path to avalanche directory + simDF: pandas dataFrame + daaframe of the configuration + specDir: str + path to a directory where simulation configuration shal be saved - optional + csvName: str + name of csv file in which to save to - optional + + Returns + -------- + configFiles: pathlib Path + path where the configuration dataframe was saved """ # collect all configuration files for this module from directory - if specDir != '': - inDir = pathlib.Path(specDir, 'configurationFiles') + if specDir != "": + inDir = pathlib.Path(specDir, "configurationFiles") else: - inDir = pathlib.Path(avaDir, 'Outputs', 'com1DFA', 'configurationFiles') + inDir = pathlib.Path(avaDir, "Outputs", "com1DFA", "configurationFiles") configFiles = inDir / csvName simDF.to_csv(configFiles) @@ -743,51 +757,51 @@ def writeAllConfigurationInfo(avaDir, simDF, specDir='', csvName='allConfigurati def convertToCfgList(parameterList): """ convert a list into a string where individual list items are separated by | - Parameters - ----------- - parameterList: list - list of parameter values + Parameters + ----------- + parameterList: list + list of parameter values - Returns - --------- - parameterString: str - str with parameter values separated by | + Returns + --------- + parameterString: str + str with parameter values separated by | """ if len(parameterList) == 0: - parameterString = '' + parameterString = "" else: parameterString = parameterList[0] for item in parameterList[1:]: - parameterString = parameterString + '|' + item + parameterString = parameterString + "|" + item return parameterString def getNumberOfProcesses(cfgMain, nSims): - """ Determine how many CPU cores to take for parallel tasks + """Determine how many CPU cores to take for parallel tasks - Parameters - ----------- - cfgMain: configuration object - the main avaframe configuration - nSims: integer - number of simulations that need to be calculated + Parameters + ----------- + cfgMain: configuration object + the main avaframe configuration + nSims: integer + number of simulations that need to be calculated - Returns - --------- - nCPU: int - number of cores to take + Returns + --------- + nCPU: int + number of cores to take """ maxCPU = multiprocessing.cpu_count() - if cfgMain["MAIN"]["nCPU"] == 'auto': - cpuPerc = float(cfgMain["MAIN"]["CPUPercent"]) / 100. + if cfgMain["MAIN"]["nCPU"] == "auto": + cpuPerc = float(cfgMain["MAIN"]["CPUPercent"]) / 100.0 nCPU = math.floor(maxCPU * cpuPerc) else: - nCPU = cfgMain['MAIN'].getint('nCPU') + nCPU = cfgMain["MAIN"].getint("nCPU") # if number of sims is lower than nCPU nCPU = min(nCPU, nSims) diff --git a/avaframe/out3Plot/plotUtils.py b/avaframe/out3Plot/plotUtils.py index b94f06400..cb4f36a6f 100644 --- a/avaframe/out3Plot/plotUtils.py +++ b/avaframe/out3Plot/plotUtils.py @@ -36,7 +36,7 @@ # Load all input Parameters from config file # get the configuration of an already imported module -cfg = cfgUtils.getModuleConfig(plotUtils) +cfg = cfgUtils.getModuleConfig(plotUtils, cfgMain['MAIN']['configurationDir']) cfgPlotUtils = cfg["UNITS"] cfgConstants = cfg["CONSTANTS"] cfg = cfg["MAIN"] diff --git a/avaframe/runCom1DFA.py b/avaframe/runCom1DFA.py index 4d72def6e..f64642d58 100644 --- a/avaframe/runCom1DFA.py +++ b/avaframe/runCom1DFA.py @@ -17,8 +17,8 @@ from avaframe.in3Utils import fileHandlerUtils as fU -def runCom1DFA(avalancheDir='', calibration=''): - """ Run com1DFA in the default configuration with only an +def runCom1DFA(avalancheDir="", configurationDir="", calibration=""): + """Run com1DFA in the default configuration with only an avalanche directory as input and the (optional) friction calibration size @@ -39,21 +39,25 @@ def runCom1DFA(avalancheDir='', calibration=''): startTime = time.time() # log file name; leave empty to use default runLog.log - logName = 'runCom1DFA' + logName = "runCom1DFA" # Load avalanche directory from general configuration file # More information about the configuration can be found here # on the Configuration page in the documentation cfgMain = cfgUtils.getGeneralConfig() - if avalancheDir != '': - cfgMain['MAIN']['avalancheDir'] = avalancheDir + if avalancheDir != "": + cfgMain["MAIN"]["avalancheDir"] = avalancheDir else: - avalancheDir = cfgMain['MAIN']['avalancheDir'] + avalancheDir = cfgMain["MAIN"]["avalancheDir"] + if configurationDir != "": + cfgMain["MAIN"]["configurationDir"] = configurationDir + else: + configurationDir = cfgMain["MAIN"]["configurationDir"] # Start logging log = logUtils.initiateLogger(avalancheDir, logName) - log.info('MAIN SCRIPT') - log.info('Current avalanche: %s', avalancheDir) + log.info("MAIN SCRIPT") + log.info("Current avalanche: %s", avalancheDir) # ---------------- # Clean input directory(ies) of old work and output files @@ -62,18 +66,18 @@ def runCom1DFA(avalancheDir='', calibration=''): initProj.cleanSingleAvaDir(avalancheDir, deleteOutput=False) # Set friction model according to cmd argument - cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, toPrint=False) - - if calibration.lower() == 'small': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATSmall' - elif calibration.lower() == 'medium': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATMedium' - elif calibration.lower() == 'large': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosAT' - elif calibration.lower() == 'auto': - cfgCom1DFA['GENERAL']['frictModel'] = 'samosATAuto' + cfgCom1DFA = cfgUtils.getModuleConfig(com1DFA, configDir=configurationDir, toPrint=False) + + if calibration.lower() == "small": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATSmall" + elif calibration.lower() == "medium": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATMedium" + elif calibration.lower() == "large": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosAT" + elif calibration.lower() == "auto": + cfgCom1DFA["GENERAL"]["frictModel"] = "samosATAuto" else: - log.info('no friction calibration override given - using ini') + log.info("no friction calibration override given - using ini") # ---------------- # Run dense flow @@ -81,25 +85,34 @@ def runCom1DFA(avalancheDir='', calibration=''): # Get peakfiles to return to QGIS avaDir = pathlib.Path(avalancheDir) - inputDir = avaDir / 'Outputs' / 'com1DFA' / 'peakFiles' + inputDir = avaDir / "Outputs" / "com1DFA" / "peakFiles" peakFilesDF = fU.makeSimDF(inputDir, avaDir=avaDir) # Print time needed endTime = time.time() - log.info('Took %6.1f seconds to calculate.' % (endTime - startTime)) + log.info("Took %6.1f seconds to calculate." % (endTime - startTime)) return peakFilesDF -if __name__ == '__main__': - - parser = argparse.ArgumentParser(description='Run com1DFA workflow') - parser.add_argument('avadir', metavar='avalancheDir', type=str, nargs='?', default='', - help='the avalanche directory') - parser.add_argument('-fc', '--friction_calibration', choices=['auto', 'large', 'medium', 'small', 'ini'], - type=str, default='ini', - help='friction calibration override, possible values are large, medium , small, auto and ini.' + - 'Overrides default AND local configs') +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Run com1DFA workflow") + parser.add_argument( + "avadir", metavar="avalancheDir", type=str, nargs="?", default="", help="the avalanche directory" + ) + parser.add_argument( + "configdir", metavar="configurationDir", type=str, nargs="?", default="", + help="the configuration file directory" + ) + parser.add_argument( + "-fc", + "--friction_calibration", + choices=["auto", "large", "medium", "small", "ini"], + type=str, + default="ini", + help="friction calibration override, possible values are large, medium , small, auto and ini." + + "Overrides default AND local configs", + ) args = parser.parse_args() - runCom1DFA(str(args.avadir), str(args.friction_calibration)) + runCom1DFA(str(args.avadir), str(args.configdir),str(args.friction_calibration)) diff --git a/avaframe/runScripts/runPlotContoursFromAsc.py b/avaframe/runScripts/runPlotContoursFromAsc.py index 04dac701f..46911606b 100644 --- a/avaframe/runScripts/runPlotContoursFromAsc.py +++ b/avaframe/runScripts/runPlotContoursFromAsc.py @@ -26,7 +26,7 @@ def plotContoursFromAsc(cfg, avalancheDir): if __name__ == "__main__": # Load configuration for runPlotContour - cfg = cfgUtils.getModuleConfig(rCon, fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) + cfg = cfgUtils.getModuleConfig(rCon, '', fileOverride="", modInfo=False, toPrint=False, onlyDefault=False) # fetch input directory cfgMain = cfgUtils.getGeneralConfig() diff --git a/docs/configuration.rst b/docs/configuration.rst index 402662068..033aeeda0 100644 --- a/docs/configuration.rst +++ b/docs/configuration.rst @@ -61,4 +61,14 @@ in the module **B** configuration file are used to update the configuration sett the configuration parameters used for one task in one configuration file. An example of this usage can be found in ``ana1Tests/energyLineTestCfg.ini``. +Additionally, there is the option to write ``local_moduleNameCfg.ini`` configuration files for all modules +where a corresponding *collectionName_moduleName_override* section is provided by using +the function: :py:func:`in3Utils.cfgHandling.rewriteLocalCfgs`. These ``local_moduleNameCfg.ini`` files are +saved to ``avalancheDir/Inputs/configurationsOverrides`` by default if no ``localCfgPath`` is provided as input. +An example of this functionality is provided in :py:mod:`runScripts.runPlotContoursFromAsc.py` with the respective +``runPlotContoursFromAscCfg.ini`` file. + +.. Note:: + By using :py:func:`in3Utils.cfgHandling.rewriteLocalCfgs`, already existing ``local_moduleNameCfg.ini`` files in the + respective directory will be overwritten.