diff --git a/Source/moja.core/include/moja/dynamic.h b/Source/moja.core/include/moja/dynamic.h index 3ed2d4e..abe8124 100644 --- a/Source/moja.core/include/moja/dynamic.h +++ b/Source/moja.core/include/moja/dynamic.h @@ -1,3 +1,5 @@ +/* + //Using POCO::DYNAMIC::VAR #pragma once #include @@ -8,3 +10,173 @@ using DynamicVar = Poco::Dynamic::Var; using DynamicVector = Poco::Dynamic::Vector; using DynamicObject = Poco::DynamicStruct; } // namespace moja + +*/ + +//Using std::any + +#pragma once + +#if 0 +#include +#include + +namespace moja { +using DynamicVar = Poco::Dynamic::Var; +using DynamicVector = Poco::Dynamic::Vector; +using DynamicObject = Poco::DynamicStruct; +} // namespace moja +#endif + +#include +#include +#include +#include +#include + +namespace moja { +class DynamicVar; + +template +class Struct { + public: + typedef typename std::map Data; + typedef typename std::set NameSet; + typedef typename Data::iterator Iterator; + typedef typename Data::const_iterator ConstIterator; + typedef typename Struct::Data::value_type ValueType; + typedef typename Struct::Data::size_type SizeType; + typedef typename std::pair::Iterator, bool> InsRetVal; + + Struct() : data_() {} + + Struct(const Data& val) : data_(val) {} + + template + explicit Struct(const std::map& val) { + typedef typename std::map::const_iterator MapConstIterator; + MapConstIterator it = val.begin(); + MapConstIterator end = val.end(); + for (; it != end; ++it) data_.insert(ValueType(it->first, Var(it->second))); + } + + virtual ~Struct() = default; + + DynamicVar& operator[](const K& name) { return data_[name]; } + + const DynamicVar& operator[](const K& name) const { + ConstIterator it = find(name); + if (it == end()) throw std::runtime_error("name not found: " + name); + return it->second; + } + + [[nodiscard]] bool contains(const K& name) const { return find(name) != end(); } + + Iterator find(const K& name) { return data_.find(name); } + + [[nodiscard]] ConstIterator find(const K& name) const { return data_.find(name); } + + Iterator end() { return data_.end(); } + + [[nodiscard]] ConstIterator end() const { return data_.end(); } + + Iterator begin() { return data_.begin(); } + + [[nodiscard]] ConstIterator begin() const { return data_.begin(); } + + template + InsRetVal insert(const K& key, const T& value) { + return data_.emplace(key, value); + } + + InsRetVal insert(const ValueType& aPair) { return data_.emplace(aPair); } + + SizeType erase(const K& key) { return data_.erase(key); } + + void erase(Iterator& it) { data_.erase(it); } + + [[nodiscard]] bool empty() const { return data_.empty(); } + + void clear() { data_.clear(); } + + [[nodiscard]] SizeType size() const { return data_.size(); } + + [[nodiscard]] NameSet members() const { + NameSet keys; + ConstIterator it = begin(); + ConstIterator itEnd = end(); + for (; it != itEnd; ++it) keys.insert(it->first); + return keys; + } + + private: + Data data_; +}; + +class DynamicVar { + public: + DynamicVar() = default; + template + DynamicVar(const T& val) : value_(val) {} + template + operator T() const { + if (!value_.has_value()) throw std::runtime_error("Can not convert empty value."); + return extract(); + } + template + const T& extract() const { + if (value_.has_value()) { + return std::any_cast(value_); + } + throw std::runtime_error("Can not extract empty value."); + } + template + DynamicVar& operator=(const T& other) + /// Assignment operator for assigning POD to DynamicVar + { + value_ = other; + return *this; + } + + [[nodiscard]] bool isEmpty() const { return !value_.has_value(); } + [[nodiscard]] const type_info& type() const { return value_.type(); } + + DynamicVar& operator[](const std::string& name); + const DynamicVar& operator[](const std::string& name) const; + bool isObject() const { return value_.type() == typeid(Struct); } + bool isVector() const { return value_.type() == typeid(std::vector); } + bool isStruct() const { return value_.type() == typeid(Struct); } + bool isInteger() const { return value_.type() == typeid(int); } + bool isString() const { return value_.type() == typeid(std::string); } + bool isNumeric() const { return value_.type() == typeid(double); } + bool isBoolean() const { return value_.type() == typeid(bool); } + bool isSigned() const { return value_.type() == typeid(signed); } + int size() const { return sizeof(value_); } + + private: + template + static DynamicVar& structIndexOperator(T* struct_pointer, N n); + DynamicVar& get_at(const std::string& name); + + std::any value_; +}; + +inline DynamicVar& DynamicVar::operator[](const std::string& name) { return get_at(name); } + +inline const DynamicVar& DynamicVar::operator[](const std::string& name) const { + return const_cast(this)->get_at(name); +} + +inline DynamicVar& DynamicVar::get_at(const std::string& name) { + return structIndexOperator(std::any_cast*>(value_), name); +} + +template +DynamicVar& DynamicVar::structIndexOperator(T* struct_pointer, N n) { + return struct_pointer->operator[](n); +} + +using DynamicObject = Struct; +using DynamicVector = std::vector; + +} // namespace moja diff --git a/Source/moja.core/include/moja/pocojsonutils.h b/Source/moja.core/include/moja/pocojsonutils.h index 5aa4334..f978bdd 100644 --- a/Source/moja.core/include/moja/pocojsonutils.h +++ b/Source/moja.core/include/moja/pocojsonutils.h @@ -12,12 +12,12 @@ namespace moja { DateTime CORE_API parseSimpleDate(const std::string& yyyymmdd); -DynamicVar CORE_API parsePocoJSONToDynamic(const DynamicVar& data); +DynamicVar CORE_API parsePocoJSONToDynamic(const Poco::Dynamic::Var& data); DynamicVector CORE_API parsePocoJSONToDynamic(const Poco::JSON::Array::Ptr& val); DynamicVar CORE_API parsePocoJSONToDynamic(const Poco::JSON::Object::Ptr& val); -DynamicVar CORE_API parsePocoVarToDynamic(const DynamicVar& var); +DynamicVar CORE_API parsePocoVarToDynamic(const Poco::Dynamic::Var& var); } // namespace moja -#endif // MOJA_CORE_POCOUTILS_H_ \ No newline at end of file +#endif // MOJA_CORE_POCOUTILS_H_ diff --git a/Source/moja.core/src/pocojsonutils.cpp b/Source/moja.core/src/pocojsonutils.cpp index d00fe9c..ffc8a89 100644 --- a/Source/moja.core/src/pocojsonutils.cpp +++ b/Source/moja.core/src/pocojsonutils.cpp @@ -156,7 +156,7 @@ DynamicVar parsePocoJSONToDynamic(const Poco::JSON::Object::Ptr& val) { // -------------------------------------------------------------------------------------------- -DynamicVar parsePocoJSONToDynamic(const DynamicVar& data) { +DynamicVar parsePocoJSONToDynamic(const Poco::Dynamic::Var& data) { if (data.isEmpty()) return DynamicVar(); if (data.type() == typeid(Poco::JSON::Object::Ptr)) { const auto& object = data.extract(); @@ -191,7 +191,7 @@ DynamicVar parsePocoJSONToDynamic(const DynamicVar& data) { // -------------------------------------------------------------------------------------------- -DynamicVar parsePocoVarToDynamic(const DynamicVar& var) { +DynamicVar parsePocoVarToDynamic(const Poco::Dynamic::Var& var) { if (var.isEmpty()) { return DynamicVar(); } diff --git a/Source/moja.datarepository/src/providerrelationalsqlite.cpp b/Source/moja.datarepository/src/providerrelationalsqlite.cpp index ce31ba8..490156d 100644 --- a/Source/moja.datarepository/src/providerrelationalsqlite.cpp +++ b/Source/moja.datarepository/src/providerrelationalsqlite.cpp @@ -100,7 +100,7 @@ class SQLiteStatement { class ProviderRelationalSQLite::impl { public: - impl(DynamicObject settings) : _path(settings["path"].convert()), _conn(_path), _cache(10000) {} + impl(DynamicObject settings) : _path(settings["path"].extract()), _conn(_path), _cache(10000) {} DynamicVar GetDataSet(const std::string& query) const { auto cachedValue = _cache.get(query); diff --git a/Source/moja.flint.configuration/include/moja/flint/configuration/json2configurationprovider.h b/Source/moja.flint.configuration/include/moja/flint/configuration/json2configurationprovider.h index d411107..f5a494d 100644 --- a/Source/moja.flint.configuration/include/moja/flint/configuration/json2configurationprovider.h +++ b/Source/moja.flint.configuration/include/moja/flint/configuration/json2configurationprovider.h @@ -34,25 +34,25 @@ class CONFIGURATION_API JSON2ConfigurationProvider : public IConfigurationProvid static bool fileExists(const std::string& path); - void createLocalDomain(DynamicVar& parsedJSON, Configuration& config) const; - void createSpinup(DynamicVar& parsedJSON, Configuration& config) const; - void createLibraries(DynamicVar& parsedJSON, Configuration& config) const; - void createProviders(DynamicVar& parsedJSON, Configuration& config) const; - void createPools(DynamicVar& parsedJSON, Configuration& config) const; + void createLocalDomain(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; + void createSpinup(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; + void createLibraries(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; + void createProviders(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; + void createPools(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; bool createSpecialPools(const std::string& poolName, const Poco::DynamicStruct& poolSettings, Configuration& config) const; - bool createSpecialVariables(const std::pair& keyValPair, Configuration& config, - bool isSpinup) const; - void createVariables(DynamicVar& parsedJSON, Configuration& config, bool isSpinup = false) const; - void createModules(DynamicVar& parsedJSON, Configuration& config) const; - void createSpinupModules(DynamicVar& parsedJSON, Configuration& config) const; - - void mergeJsonConfigFile(std::string fileName, DynamicVar parsedResult); - void mergeJsonProviderConfigFile(std::string fileName, DynamicVar parsedResult); + bool createSpecialVariables(const std::pair& keyValPair, + Configuration& config, bool isSpinup) const; + void createVariables(Poco::Dynamic::Var& parsedJSON, Configuration& config, bool isSpinup = false) const; + void createModules(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; + void createSpinupModules(Poco::Dynamic::Var& parsedJSON, Configuration& config) const; + + void mergeJsonConfigFile(std::string fileName, Poco::Dynamic::Var parsedResult); + void mergeJsonProviderConfigFile(std::string fileName, Poco::Dynamic::Var parsedResult); }; } // namespace configuration } // namespace flint } // namespace moja -#endif // MOJA_FLINT_CONFIGURATION_JSON2CONFIGURATIONPROVIDER_H_ +#endif // MOJA_FLINT_CONFIGURATION_JSON2CONFIGURATIONPROVIDER_H_ \ No newline at end of file diff --git a/Source/moja.flint.configuration/src/json2configurationprovider.cpp b/Source/moja.flint.configuration/src/json2configurationprovider.cpp index ea38a7d..ad7dcc3 100644 --- a/Source/moja.flint.configuration/src/json2configurationprovider.cpp +++ b/Source/moja.flint.configuration/src/json2configurationprovider.cpp @@ -1,734 +1,761 @@ -#include "moja/flint/configuration/json2configurationprovider.h" - -#include "moja/flint/configuration/configurationexceptions.h" -#include "moja/flint/configuration/iterationbase.h" -#include "moja/flint/configuration/iterationtileindex.h" -#include "moja/flint/configuration/library.h" -#include "moja/flint/configuration/localdomain.h" -#include "moja/flint/configuration/spinup.h" - -#include -#include -#include - -#include -#include -#include -#include -#include - -#include -#include - -using Poco::DynamicStruct; -using Poco::Dynamic::Var; -using Poco::JSON::Object; -using Poco::JSON::ParseHandler; -using Poco::JSON::Parser; -using Poco::JSON::Stringifier; - -#include - -namespace moja { -namespace flint { -namespace configuration { - -JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath) { - MOJA_PROFILE_FUNCTION(); - for (const std::string configFie : configFilePath) { - Poco::File file(configFie); - if (!file.exists()) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); - } - } - _configFilePath = configFilePath; - //_configProviderFilePath = null; - _hasProviderConfigFile = false; -} - -JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath, - const std::vector& configProviderFilePath) { - MOJA_PROFILE_FUNCTION(); - for (const auto& configFie : configFilePath) { - Poco::File file(configFie); - if (!file.exists()) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); - } - } - _configFilePath = std::move(configFilePath); - - if (!configProviderFilePath.empty()) { - for (const auto& configProviderFile : configProviderFilePath) { - Poco::File fileProvider(configProviderFile); - if (!fileProvider.exists()) { - BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configProviderFile)); - } - } - _configProviderFilePath = std::move(configProviderFilePath); - _hasProviderConfigFile = true; - } else { - //_configProviderFilePath = ""; - _hasProviderConfigFile = false; - } -} - -std::shared_ptr JSON2ConfigurationProvider::createConfiguration() { - MOJA_PROFILE_FUNCTION(); - // parse the first run configuration file - auto fileName = _configFilePath[0]; - std::ifstream file(fileName); - Poco::JSON::Parser jsonParser; - auto parsedResult = jsonParser.parse(file); - - // parse and merge rest run configuration files - for (int index = 1; index < _configFilePath.size(); index++) { - auto curFileName = _configFilePath[index]; - mergeJsonConfigFile(curFileName, parsedResult); - } - - Poco::DynamicStruct jsonStruct = *parsedResult.extract(); - auto localDomainStruct = jsonStruct["LocalDomain"].extract(); - auto start = parseSimpleDate(localDomainStruct["start_date"].extract()); - auto end = parseSimpleDate(localDomainStruct["end_date"].extract()); - - auto config = std::make_shared(start, end); - - if (_hasProviderConfigFile) { - Poco::JSON::Parser jsonProviderParser; - std::ifstream fileProvider(_configProviderFilePath[0]); - - // parse the first provider configuration file - auto providerParsedJSON = jsonProviderParser.parse(fileProvider); - auto providerParsedResult = jsonProviderParser.result(); - - // parse and merge the reset provider configuration files - for (int index = 1; index < _configProviderFilePath.size(); index++) { - auto curProviderFileName = _configProviderFilePath[index]; - mergeJsonProviderConfigFile(curProviderFileName, providerParsedResult); - } - - createProviders(providerParsedResult, *config); - } - - createLocalDomain(parsedResult, *config); - createSpinup(parsedResult, *config); - createLibraries(parsedResult, *config); - createPools(parsedResult, *config); - createVariables(parsedResult, *config); - createModules(parsedResult, *config); - if (config->spinup()->enabled()) { // only expect this list if spinup enabled - createSpinupModules(parsedResult, *config); - createVariables(parsedResult, *config, true); - } - - return config; -} - -/* -Parse and merge run configuration files. -It only merges up to 2nd level configuration. -*/ -void JSON2ConfigurationProvider::mergeJsonConfigFile(std::string fileName, DynamicVar parsedResult) { - MOJA_PROFILE_FUNCTION(); - auto jsonStructFinal = parsedResult.extract(); - std::ifstream file(fileName); - - if (file.is_open()) { - Poco::JSON::Parser jsonParser; - - auto currentParsedResult = jsonParser.parse(file); - auto jsonStructCurrent = currentParsedResult.extract(); - - std::vector names; - jsonStructCurrent->getNames(names); - - for (std::string name : names) { - auto definedInParsedResult = jsonStructFinal->has(name); - auto currentConfigObj = jsonStructCurrent->get(name); - if (!definedInParsedResult) { - // insert the undefined config struct - jsonStructFinal->set(name, currentConfigObj); - } else { - auto finalObj = jsonStructFinal->get(name); - auto finalDynamicObj = parsePocoVarToDynamic(finalObj); - auto finalDynamicObjContents = - finalDynamicObj.isEmpty() ? DynamicObject() : finalDynamicObj.extract(); - - auto currentDynamicObj = parsePocoVarToDynamic(currentConfigObj); - auto currentObjContents = - currentDynamicObj.isEmpty() ? DynamicObject() : currentDynamicObj.extract(); - - for (const auto& item : currentObjContents) { - auto itemName = std::get<0>(item); - auto findItem = finalDynamicObjContents.find(itemName); - - // not found, insert the config pair - if (findItem == finalDynamicObjContents.end()) { - auto finalObjPtr = finalObj.extract(); - finalObjPtr->set(itemName, std::get<1>(item)); - } - } - } - } - file.close(); - } -} - -/* -Parse and merge data provider configuration files -it merges only raster-tiled layers configuration. It assumes that other configurations are identical. -*/ -void JSON2ConfigurationProvider::mergeJsonProviderConfigFile(std::string fileName, DynamicVar parsedResult) { - MOJA_PROFILE_FUNCTION(); - auto jsonStructFinal = parsedResult.extract(); - auto providerFinal = jsonStructFinal->getObject("Providers"); - std::ifstream file(fileName); - - if (file.is_open()) { - Poco::JSON::Parser jsonParser; - - auto currentParsedResult = jsonParser.parse(file); - auto jsonStructCurrent = currentParsedResult.extract(); - auto providerCurrent = jsonStructCurrent->getObject("Providers"); - - std::vector names; - providerCurrent->getNames(names); - - for (auto name : names) { - bool definedInParsedResult = providerFinal->has(name); - auto currentConfigObj = providerCurrent->get(name); - if (!definedInParsedResult) { - // insert the undefined config struct - providerFinal->set(name, currentConfigObj); - } else { - auto finalObj = providerFinal->get(name); - auto finalDynamicObj = parsePocoVarToDynamic(finalObj); - auto finalDynamicObjContents = - finalDynamicObj.isEmpty() ? DynamicObject() : finalDynamicObj.extract(); - - auto currentDynamicObj = parsePocoVarToDynamic(currentConfigObj); - auto currentObjContents = - currentDynamicObj.isEmpty() ? DynamicObject() : currentDynamicObj.extract(); - - for (const auto& item : currentObjContents) { - auto itemName = std::get<0>(item); - if (itemName.compare("layers") == 0) { - auto definedLayer = finalDynamicObjContents.find("layers"); - auto ptrArr = finalObj.extract(); - auto layObj = ptrArr->get("layers"); - auto ptrArrFinal = layObj.extract(); - - auto currentLayerObj = currentConfigObj.extract(); - auto currentLayers = currentLayerObj->get("layers").extract(); - auto sizeCur = currentLayers->size(); - for (auto i = 0; i < currentLayers->size(); i++) { - auto value = currentLayers->get(i); - ptrArrFinal->add(value); - } - } else { - auto findItem = finalDynamicObjContents.find(itemName); - - if (findItem == finalDynamicObjContents.end()) { - // not found, insert the new config pair - auto finalObjPtr = finalObj.extract(); - finalObjPtr->set(itemName, std::get<1>(item)); - } - } - } - } - } - file.close(); - } -} - -void JSON2ConfigurationProvider::createLocalDomain(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); - auto localDomainStruct = jsonStruct["LocalDomain"].extract(); - const auto localDomainType = parseLocalDomainType(localDomainStruct["type"]); - const auto sequencerLibrary = localDomainStruct["sequencer_library"].extract(); - const auto sequencer = localDomainStruct["sequencer"].extract(); - const auto simulateLandUnit = localDomainStruct["simulateLandUnit"].extract(); - const auto landUnitBuildSuccess = localDomainStruct["landUnitBuildSuccess"].extract(); - auto stepping = TimeStepping::Monthly; - if (localDomainStruct.contains("timing")) { - const auto timing = localDomainStruct["timing"].extract(); - if (timing == "annual") { - stepping = TimeStepping::Annual; - } - } - - DynamicObject operationManagerSettings; - if (localDomainStruct.contains("operationManager")) { - auto s = localDomainStruct["operationManager"]; - if (s.isStruct()) { - auto d = parsePocoVarToDynamic(localDomainStruct["operationManager"]); - operationManagerSettings = d.isEmpty() ? DynamicObject() : d.extract(); - } else { - operationManagerSettings["name"] = s.extract(); // operationManagerName; - } - } else { - operationManagerSettings["name"] = ""; - } - - DynamicObject settings; - if (localDomainStruct.contains("settings")) { - auto d = parsePocoVarToDynamic(localDomainStruct["settings"]); - settings = d.isEmpty() ? DynamicObject() : d.extract(); - } - - if (localDomainType != LocalDomainType::Point) { - auto d = parsePocoVarToDynamic(localDomainStruct["landscape"]); - auto& landscape = d.isEmpty() ? DynamicObject() : d.extract(); - - auto iterationType = - LocalDomainIterationType::LandscapeTiles; // TODO: confirm this - I've set this as the default for now - if (landscape.contains("iteration_type")) { - const auto& str = landscape["iteration_type"]; - iterationType = convertStrToLocalDomainIterationType(str); - } - - auto doLogging = true; - if (landscape.contains("do_logging")) { - doLogging = landscape["do_logging"]; - } - - auto numThreads = 1; - if (landscape.contains("num_threads")) { - numThreads = landscape["num_threads"]; - } - - config.setLocalDomain(localDomainType, iterationType, doLogging, numThreads, sequencerLibrary, sequencer, - simulateLandUnit, landUnitBuildSuccess, settings, stepping); - - config.localDomain()->setLandscapeObject(landscape["provider"].convert(), iterationType); - - auto landscapeObject = config.localDomain()->landscapeObject(); - switch (iterationType) { - case LocalDomainIterationType::AreaOfInterest: { - // NEW STUFF - landscapeObject->setIterationAreaOfInterest(); - auto aoiIteration = landscapeObject->iterationAreaOfInterest(); - // TODO: fill this in - break; - } - case LocalDomainIterationType::ASpatialIndex: { - landscapeObject->setIterationASpatialIndex(); - auto asiIteration = landscapeObject->iterationASpatialIndex(); - asiIteration->set_maxTileSize(landscape["max_tile_size"]); - asiIteration->set_tileCacheSize(landscape["tile_cache_size"]); - break; - } - case LocalDomainIterationType::ASpatialMongoIndex: { - landscapeObject->setIterationASpatialMongoIndex(); - auto asmiIteration = landscapeObject->iterationASpatialMongoIndex(); - - if (landscape.contains("process_all")) asmiIteration->set_processAlldocuments(landscape["process_all"]); - - if (landscape.contains("documents")) { - for (auto& item : landscape["documents"].extract()) { - std::string id = item["id"]; - asmiIteration->addDocumentId(id); - } - } - break; - } - case LocalDomainIterationType::LandscapeTiles: { - landscapeObject->setIterationLandscapeTiles(); - auto ltIteration = landscapeObject->iterationLandscapeTiles(); - ltIteration->set_tileSizeX(landscape["tile_size_x"]); - ltIteration->set_tileSizeY(landscape["tile_size_y"]); - ltIteration->set_xPixels(landscape["x_pixels"]); - ltIteration->set_yPixels(landscape["y_pixels"]); - for (auto& tileConfig : landscape["tiles"].extract()) { - int xIndex = tileConfig["x"]; - int yIndex = tileConfig["y"]; - ltIteration->landscapeTiles().push_back(ConfigTile(xIndex, yIndex, ltIteration->tileSizeX(), - ltIteration->tileSizeY(), ltIteration->xPixels(), - ltIteration->yPixels())); - } - break; - } - case LocalDomainIterationType::TileIndex: { - landscapeObject->setIterationTileIndex(); - auto tileIteration = landscapeObject->iterationTileIndex(); - - for (auto& item : landscape["tiles"].extract()) { - UInt32 tileIdx = item["tile_index"]; - tileIteration->addTileIndex(ConfigTileIdx(tileIdx)); - } - break; - } - case LocalDomainIterationType::BlockIndex: { - landscapeObject->setIterationBlockIndex(); - auto blockIteration = landscapeObject->iterationBlockIndex(); - for (auto& item : landscape["blocks"].extract()) { - const UInt32 tileIdx = item["tile_index"]; - const UInt32 blockIdx = item["block_index"]; - blockIteration->addBlockIndex(ConfigBlockIdx(tileIdx, blockIdx)); - } - break; - } - case LocalDomainIterationType::CellIndex: { - landscapeObject->setIterationCellIndex(); - auto cellIteration = landscapeObject->iterationCellIndex(); - for (auto& item : landscape["cells"].extract()) { - const UInt32 tileIdx = item["tile_index"]; - const UInt32 blockIdx = item["block_index"]; - const UInt32 cellIdx = item["cell_index"]; - cellIteration->addCellIndex(ConfigCellIdx(tileIdx, blockIdx, cellIdx)); - } - break; - } - case LocalDomainIterationType::NotAnIteration: - break; - default: - break; - } - } else { - config.setLocalDomain(localDomainType, LocalDomainIterationType::NotAnIteration, true, 1, sequencerLibrary, - sequencer, simulateLandUnit, landUnitBuildSuccess, settings); - } - config.localDomain()->operationManagerObject()->set_name( - operationManagerSettings["name"].extract()); - config.localDomain()->operationManagerObject()->set_settings(operationManagerSettings); -} - -void JSON2ConfigurationProvider::createSpinup(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); - if (!jsonStruct.contains("Spinup")) { - DynamicObject settings; - config.setSpinup(false, "NA", "NA", "NA", "NA", settings); - return; - } - - auto spinupStruct = jsonStruct["Spinup"].extract(); - bool enabled = false; - if (spinupStruct.contains("enabled")) { - enabled = spinupStruct["enabled"].extract(); - } - - if (enabled) { - for (auto param : {"sequencer_library", "sequencer", "simulateLandUnit", "landUnitBuildSuccess"}) { - if (!spinupStruct.contains(param)) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param((boost::format("Spinup.%1%") % param).str())); - } - } - - auto sequencerLibrary = spinupStruct["sequencer_library"].extract(); - auto sequencer = spinupStruct["sequencer"].extract(); - auto simulateLandUnit = spinupStruct["simulateLandUnit"].extract(); - auto landUnitBuildSuccess = spinupStruct["landUnitBuildSuccess"].extract(); - auto d = parsePocoVarToDynamic(jsonStruct["Spinup"]); - auto& settings = d.isEmpty() ? DynamicObject() : d.extract(); - config.setSpinup(enabled, sequencerLibrary, sequencer, simulateLandUnit, landUnitBuildSuccess, settings); - } else { - DynamicObject settings; - config.setSpinup(false, "NA", "NA", "NA", "NA", settings); - } -} - -void JSON2ConfigurationProvider::createLibraries(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); - auto librariesStruct = jsonStruct["Libraries"].extract(); - for (auto item : librariesStruct) { - if (item.second.isStruct()) { - auto libraryStruct = item.second.extract(); - auto libraryName = libraryStruct["library"].extract(); - auto libraryPath = libraryStruct["path"].extract(); // item.first; - const auto libraryTypeStr = libraryStruct["type"].extract(); - - MOJA_LOG_DEBUG << "details (" << item.first << "): path - " << libraryPath << ", filename -" << libraryName; - - config.addLibrary(libraryPath, libraryName, item.first, parseLibraryType(libraryTypeStr)); - } else { - config.addLibrary(item.first, parseLibraryType(item.second.extract())); - } - } -} - -bool JSON2ConfigurationProvider::fileExists(const std::string& path) { - MOJA_PROFILE_FUNCTION(); - Poco::File pf(path); - return pf.exists(); -} - -void JSON2ConfigurationProvider::createProviders(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - auto jsonStruct2 = *parsedJSON.extract(); - auto provider = jsonStruct2.getObject("Providers"); - auto& data = *(provider.get()); - for (auto& item : data) { - auto d = parsePocoJSONToDynamic(item.second); - if (d.isEmpty()) { - BOOST_THROW_EXCEPTION(ProviderSettingsException() << ProviderName(item.first)); - } - auto xx = d.extract(); - std::string libName; - if (xx.contains("library")) - libName = d["library"].convert(); - else - libName = "internal.flint"; - - const std::string providerType = d["type"].convert(); - auto& settings = d.extract(); - config.addProvider(item.first, libName, providerType, settings); - } -} - -void JSON2ConfigurationProvider::createPools(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); - auto poolsItem = jsonStruct["Pools"]; - - if (poolsItem.isStruct()) { - auto poolsStruct = poolsItem.extract(); - for (auto item : poolsStruct) { - if (item.second.isStruct()) { - auto poolStruct = item.second.extract(); - if (createSpecialPools(item.first, poolStruct, config)) { - continue; - } - - auto poolDesc = poolStruct["description"].extract(); - auto poolUnits = poolStruct["units"].extract(); - auto poolScale = poolStruct["scale"].extract(); - auto poolOrder = int(poolStruct["order"]); - config.addPool(item.first, poolDesc, poolUnits, poolScale, poolOrder, item.second.extract()); - } else { - auto v = parsePocoVarToDynamic(item.second); - config.addPool(item.first, item.second.extract()); - } - } - } else if (poolsItem.isArray()) { - auto poolsArr = poolsItem.extract(); - for (auto item : poolsArr) { - if (item.isStruct()) { - auto poolStruct = item.extract(); - if (poolStruct.size() == 1) { - auto poolName = poolStruct.begin()->first; - if (createSpecialPools(poolName, poolStruct, config)) { - continue; - } - - auto poolValue = poolStruct.begin()->second.extract(); - config.addPool(poolName, poolValue); - } else { - auto poolName = poolStruct["name"].extract(); - if (createSpecialPools(poolName, poolStruct, config)) { - continue; - } - - auto poolDesc = poolStruct["description"].extract(); - auto poolUnits = poolStruct["units"].extract(); - auto poolScale = poolStruct["scale"].extract(); - auto poolOrder = int(poolStruct["order"]); - auto poolValue = poolStruct["value"].extract(); - config.addPool(poolName, poolDesc, poolUnits, poolScale, poolOrder, poolValue); - } - } else { - // TODO: Check this, perhaps we just have a list of Pool names to set to 0? - const auto poolName = item.extract(); - config.addPool(poolName, 0.0); - } - } - } -} - -bool JSON2ConfigurationProvider::createSpecialPools(const std::string& poolName, const DynamicStruct& poolSettings, - Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - if (!poolSettings.contains("transform")) { - return false; - } - - auto poolDesc = poolSettings.contains("description") ? poolSettings["description"].extract() : ""; - auto poolUnits = poolSettings.contains("units") ? poolSettings["units"].extract() : ""; - auto poolScale = poolSettings.contains("scale") ? poolSettings["scale"].extract() : 1.0; - auto poolOrder = poolSettings.contains("order") ? int(poolSettings["order"]) : 0; - - auto transformSettings = parsePocoVarToDynamic(poolSettings["transform"]); - auto settings = transformSettings.extract(); - auto typeName = transformSettings["type"].extract(); - auto libraryName = transformSettings["library"].extract(); - - config.addExternalPool(poolName, poolDesc, poolUnits, poolScale, poolOrder, libraryName, typeName, settings); - - return true; -} - -bool JSON2ConfigurationProvider::createSpecialVariables(const std::pair& keyValPair, - Configuration& config, bool isSpinup) const { - MOJA_PROFILE_FUNCTION(); - auto spinup = config.spinup(); - auto varName = keyValPair.first; - if (keyValPair.second.isStruct()) { - auto ss = keyValPair.second.extract(); - if (ss.size() == 1) { - auto keyValPairSS = ss.begin(); - if (keyValPairSS->first == "flintdata") { - auto v = parsePocoVarToDynamic(keyValPairSS->second); - auto varStruct = v.extract(); - auto typeName = varStruct["type"].extract(); - auto libraryName = varStruct["library"].extract(); - auto& settings = - varStruct.contains("settings") ? varStruct["settings"].extract() : DynamicObject(); - isSpinup ? spinup->addFlintDataVariable(varName, libraryName, typeName, settings) - : config.addFlintDataVariable(varName, libraryName, typeName, settings); - return true; - } - if (keyValPairSS->first == "transform") { - auto v = parsePocoVarToDynamic(keyValPairSS->second); - auto varStruct = v.extract(); - auto typeName = varStruct["type"].extract(); - auto libraryName = varStruct["library"].extract(); - isSpinup ? spinup->addExternalVariable(varName, libraryName, typeName, varStruct) - : config.addExternalVariable(varName, libraryName, typeName, varStruct); - return true; - } - } - } - return false; -} - -void JSON2ConfigurationProvider::createVariables(DynamicVar& parsedJSON, Configuration& config, bool isSpinup) const { - MOJA_PROFILE_FUNCTION(); - auto spinup = config.spinup(); - const std::string configSection = isSpinup ? "SpinupVariables" : "Variables"; - - Poco::DynamicStruct jsonStruct2 = *parsedJSON.extract(); - auto& variablesItemVar = jsonStruct2[configSection]; - - if (variablesItemVar.isStruct()) { - const auto& variablesItem = variablesItemVar.extract(); - int index = 0; - for (const auto& keyValPair : variablesItem) { - index++; - const auto varName = keyValPair.first; - if (createSpecialVariables(keyValPair, config, isSpinup)) continue; - auto v = parsePocoVarToDynamic(keyValPair.second); - isSpinup ? spinup->addVariable(varName, v) : config.addVariable(varName, v); - } - } else if (variablesItemVar.isArray()) { - auto arr = variablesItemVar.extract(); - - int index = 0; - for (auto& value : arr) { - index++; - if (value.isStruct()) { - const auto& s = value.extract(); - if (value.size() == 1) { - auto keyValPair = s.begin(); - auto varName = keyValPair->first; - if (createSpecialVariables(*keyValPair, config, isSpinup)) continue; - auto v = parsePocoVarToDynamic(keyValPair->second); - isSpinup ? spinup->addVariable(keyValPair->first, v) : config.addVariable(keyValPair->first, v); - } else { - MOJA_LOG_ERROR << "Variable defined in array (index " << index - << ") has has struct with more than one item defined)"; - } - } else { - // TODO: do we set a default value or ignore this Variable? I'd say log error and move on - auto varName = value.extract(); - MOJA_LOG_ERROR << "Variable defined in array (index " << index - << ") has no value (not Struct or Array): " << varName; - } - } - } -} - -void JSON2ConfigurationProvider::createModules(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); - auto modulesStruct = jsonStruct["Modules"].extract(); - for (auto item : modulesStruct) { - auto moduleStruct = item.second.extract(); - if (moduleStruct.contains("enabled")) { - const bool enabled = moduleStruct["enabled"].extract(); - if (!enabled) continue; - } - if (!moduleStruct.contains("library")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); - } - if (!moduleStruct.contains("order")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); - } - - const auto& moduleLibraryName = moduleStruct["library"].extract(); - auto moduleOrder = int(moduleStruct["order"]); - - bool isProxy = false; - if (moduleStruct.contains("is_proxy")) { - isProxy = moduleStruct["is_proxy"].extract(); - } - - const auto& d = parsePocoVarToDynamic(moduleStruct["settings"]); - auto& settings = d.isEmpty() ? DynamicObject() : d.extract(); - config.addModule(moduleLibraryName, item.first, moduleOrder, isProxy, settings); - } -} - -void JSON2ConfigurationProvider::createSpinupModules(DynamicVar& parsedJSON, Configuration& config) const { - MOJA_PROFILE_FUNCTION(); - Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); - auto modulesStruct = jsonStruct["SpinupModules"].extract(); - for (auto item : modulesStruct) { - auto moduleStruct = item.second.extract(); - - if (moduleStruct.contains("enabled")) { - bool enabled = moduleStruct["enabled"].extract(); - if (!enabled) continue; - } - - if (!moduleStruct.contains("library")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); - } - if (!moduleStruct.contains("order")) { - BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); - } - - const auto moduleLibraryName = moduleStruct["library"].extract(); - auto moduleOrder = int(moduleStruct["order"]); - auto moduleCreateNew = moduleStruct.contains("create_new") ? moduleStruct["create_new"].extract() : false; - - const auto& d = parsePocoVarToDynamic(moduleStruct["settings"]); - auto& settings = d.isEmpty() ? DynamicObject() : d.extract(); - auto spinup = config.spinup(); - spinup->addSpinupModule(moduleLibraryName, item.first, moduleOrder, moduleCreateNew, settings); - } -} - -LocalDomainType JSON2ConfigurationProvider::parseLocalDomainType(const std::string& type) { - if (type == "spatial_tiled") { - return LocalDomainType::SpatialTiled; - } else if (type == "spatially_referenced_sql") { - return LocalDomainType::SpatiallyReferencedSQL; - } else if (type == "spatially_referenced_nosql") { - return LocalDomainType::SpatiallyReferencedNoSQL; - } else if (type == "threaded_spatially_referenced_nosql") { - return LocalDomainType::ThreadedSpatiallyReferencedNoSQL; - } else if (type == "point") { - return LocalDomainType::Point; - } - throw std::runtime_error("Invalid LocalDomain Type specified"); -} - -LibraryType JSON2ConfigurationProvider::parseLibraryType(const std::string& type) { - if (type == "internal") { - return LibraryType::Internal; - } else if (type == "external") { - return LibraryType::External; - } else if (type == "managed") { - return LibraryType::Managed; - } else if (type == "python") { - return LibraryType::Python; - } - - throw std::runtime_error("TODO: specific exception here"); -} - -} // namespace configuration -} // namespace flint -} // namespace moja +#include "moja/flint/configuration/json2configurationprovider.h" + +#include "moja/flint/configuration/configurationexceptions.h" +#include "moja/flint/configuration/iterationbase.h" +#include "moja/flint/configuration/iterationtileindex.h" +#include "moja/flint/configuration/library.h" +#include "moja/flint/configuration/localdomain.h" +#include "moja/flint/configuration/spinup.h" + +#include +#include +#include + +#include +#include +#include +#include +#include + +#include +#include + +using Poco::DynamicStruct; +using Poco::Dynamic::Var; +using Poco::JSON::Object; +using Poco::JSON::ParseHandler; +using Poco::JSON::Parser; +using Poco::JSON::Stringifier; + +#include + +namespace moja { +namespace flint { +namespace configuration { + +JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath) { + MOJA_PROFILE_FUNCTION(); + for (const std::string configFie : configFilePath) { + Poco::File file(configFie); + if (!file.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); + } + } + _configFilePath = configFilePath; + //_configProviderFilePath = null; + _hasProviderConfigFile = false; +} + +JSON2ConfigurationProvider::JSON2ConfigurationProvider(const std::vector& configFilePath, + const std::vector& configProviderFilePath) { + MOJA_PROFILE_FUNCTION(); + for (const auto& configFie : configFilePath) { + Poco::File file(configFie); + if (!file.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configFie)); + } + } + _configFilePath = std::move(configFilePath); + + if (!configProviderFilePath.empty()) { + for (const auto& configProviderFile : configProviderFilePath) { + Poco::File fileProvider(configProviderFile); + if (!fileProvider.exists()) { + BOOST_THROW_EXCEPTION(FileNotFoundException() << FileName(configProviderFile)); + } + } + _configProviderFilePath = std::move(configProviderFilePath); + _hasProviderConfigFile = true; + } else { + //_configProviderFilePath = ""; + _hasProviderConfigFile = false; + } +} + +std::shared_ptr JSON2ConfigurationProvider::createConfiguration() { + MOJA_PROFILE_FUNCTION(); + // parse the first run configuration file + auto fileName = _configFilePath[0]; + std::ifstream file(fileName); + Poco::JSON::Parser jsonParser; + auto parsedResult = jsonParser.parse(file); + + // parse and merge rest run configuration files + for (int index = 1; index < _configFilePath.size(); index++) { + auto curFileName = _configFilePath[index]; + mergeJsonConfigFile(curFileName, parsedResult); + } + + Poco::DynamicStruct jsonStruct = *parsedResult.extract(); + auto localDomainStruct = jsonStruct["LocalDomain"].extract(); + auto start = parseSimpleDate(localDomainStruct["start_date"].extract()); + auto end = parseSimpleDate(localDomainStruct["end_date"].extract()); + + auto config = std::make_shared(start, end); + + if (_hasProviderConfigFile) { + Poco::JSON::Parser jsonProviderParser; + std::ifstream fileProvider(_configProviderFilePath[0]); + + // parse the first provider configuration file + auto providerParsedJSON = jsonProviderParser.parse(fileProvider); + auto providerParsedResult = jsonProviderParser.result(); + + // parse and merge the reset provider configuration files + for (int index = 1; index < _configProviderFilePath.size(); index++) { + auto curProviderFileName = _configProviderFilePath[index]; + mergeJsonProviderConfigFile(curProviderFileName, providerParsedResult); + } + + createProviders(providerParsedResult, *config); + } + + createLocalDomain(parsedResult, *config); + createSpinup(parsedResult, *config); + createLibraries(parsedResult, *config); + createPools(parsedResult, *config); + createVariables(parsedResult, *config); + createModules(parsedResult, *config); + if (config->spinup()->enabled()) { // only expect this list if spinup enabled + createSpinupModules(parsedResult, *config); + createVariables(parsedResult, *config, true); + } + + return config; +} + +/* +Parse and merge run configuration files. +It only merges up to 2nd level configuration. +*/ + +void JSON2ConfigurationProvider::mergeJsonConfigFile(std::string fileName, Poco::Dynamic::Var parsedResult) { + + MOJA_PROFILE_FUNCTION(); + auto jsonStructFinal = parsedResult.extract(); + std::ifstream file(fileName); + + if (file.is_open()) { + Poco::JSON::Parser jsonParser; + + auto currentParsedResult = jsonParser.parse(file); + auto jsonStructCurrent = currentParsedResult.extract(); + + std::vector names; + jsonStructCurrent->getNames(names); + + for (std::string name : names) { + auto definedInParsedResult = jsonStructFinal->has(name); + auto currentConfigObj = jsonStructCurrent->get(name); + if (!definedInParsedResult) { + // insert the undefined config struct + jsonStructFinal->set(name, currentConfigObj); + } else { + auto finalObj = jsonStructFinal->get(name); + auto finalDynamicObj = parsePocoVarToDynamic(finalObj); + auto finalDynamicObjContents = finalDynamicObj.isEmpty() + ? Poco::DynamicStruct() + : finalDynamicObj.extract(); + + auto currentDynamicObj = parsePocoVarToDynamic(currentConfigObj); + auto currentObjContents = currentDynamicObj.isEmpty() + ? Poco::DynamicStruct() + : currentDynamicObj.extract(); + + for (const auto& item : currentObjContents) { + auto itemName = std::get<0>(item); + auto findItem = finalDynamicObjContents.find(itemName); + + // not found, insert the config pair + if (findItem == finalDynamicObjContents.end()) { + auto finalObjPtr = finalObj.extract(); + finalObjPtr->set(itemName, std::get<1>(item)); + } + } + } + } + file.close(); + } +} + +/* +Parse and merge data provider configuration files +it merges only raster-tiled layers configuration. It assumes that other configurations are identical. +*/ + +void JSON2ConfigurationProvider::mergeJsonProviderConfigFile(std::string fileName, Poco::Dynamic::Var parsedResult) { + + MOJA_PROFILE_FUNCTION(); + auto jsonStructFinal = parsedResult.extract(); + auto providerFinal = jsonStructFinal->getObject("Providers"); + std::ifstream file(fileName); + + if (file.is_open()) { + Poco::JSON::Parser jsonParser; + + auto currentParsedResult = jsonParser.parse(file); + auto jsonStructCurrent = currentParsedResult.extract(); + auto providerCurrent = jsonStructCurrent->getObject("Providers"); + + std::vector names; + providerCurrent->getNames(names); + + for (auto name : names) { + bool definedInParsedResult = providerFinal->has(name); + auto currentConfigObj = providerCurrent->get(name); + if (!definedInParsedResult) { + // insert the undefined config struct + providerFinal->set(name, currentConfigObj); + } else { + auto finalObj = providerFinal->get(name); + auto finalDynamicObj = parsePocoVarToDynamic(finalObj); + auto finalDynamicObjContents = finalDynamicObj.isEmpty() + ? Poco::DynamicStruct() + : finalDynamicObj.extract(); + + auto currentDynamicObj = parsePocoVarToDynamic(currentConfigObj); + auto currentObjContents = currentDynamicObj.isEmpty() + ? Poco::DynamicStruct() + : currentDynamicObj.extract(); + + for (const auto& item : currentObjContents) { + auto itemName = std::get<0>(item); + if (itemName.compare("layers") == 0) { + auto definedLayer = finalDynamicObjContents.find("layers"); + auto ptrArr = finalObj.extract(); + auto layObj = ptrArr->get("layers"); + auto ptrArrFinal = layObj.extract(); + + auto currentLayerObj = currentConfigObj.extract(); + auto currentLayers = currentLayerObj->get("layers").extract(); + auto sizeCur = currentLayers->size(); + for (auto i = 0; i < currentLayers->size(); i++) { + auto value = currentLayers->get(i); + ptrArrFinal->add(value); + } + } else { + auto findItem = finalDynamicObjContents.find(itemName); + + if (findItem == finalDynamicObjContents.end()) { + // not found, insert the new config pair + auto finalObjPtr = finalObj.extract(); + finalObjPtr->set(itemName, std::get<1>(item)); + } + } + } + } + } + file.close(); + } +} + + +void JSON2ConfigurationProvider::createLocalDomain(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); + auto localDomainStruct = jsonStruct["LocalDomain"].extract(); + const auto localDomainType = parseLocalDomainType(localDomainStruct["type"]); + const auto sequencerLibrary = localDomainStruct["sequencer_library"].extract(); + const auto sequencer = localDomainStruct["sequencer"].extract(); + const auto simulateLandUnit = localDomainStruct["simulateLandUnit"].extract(); + const auto landUnitBuildSuccess = localDomainStruct["landUnitBuildSuccess"].extract(); + auto stepping = TimeStepping::Monthly; + if (localDomainStruct.contains("timing")) { + const auto timing = localDomainStruct["timing"].extract(); + if (timing == "annual") { + stepping = TimeStepping::Annual; + } + } + + DynamicObject operationManagerSettings; + if (localDomainStruct.contains("operationManager")) { + auto s = localDomainStruct["operationManager"]; + if (s.isStruct()) { + auto d = parsePocoVarToDynamic(localDomainStruct["operationManager"]); + operationManagerSettings = d.isEmpty() ? DynamicObject() : d.extract(); + } else { + operationManagerSettings["name"] = s.extract(); // operationManagerName; + } + } else { + operationManagerSettings["name"] = ""; + } + + DynamicObject settings; + if (localDomainStruct.contains("settings")) { + auto d = parsePocoVarToDynamic(localDomainStruct["settings"]); + settings = d.isEmpty() ? DynamicObject() : d.extract(); + } + + if (localDomainType != LocalDomainType::Point) { + auto d = parsePocoVarToDynamic(localDomainStruct["landscape"]); + auto& landscape = d.isEmpty() ? DynamicObject() : d.extract(); + + auto iterationType = + LocalDomainIterationType::LandscapeTiles; // TODO: confirm this - I've set this as the default for now + if (landscape.contains("iteration_type")) { + const auto& str = landscape["iteration_type"]; + iterationType = convertStrToLocalDomainIterationType(str); + } + + auto doLogging = true; + if (landscape.contains("do_logging")) { + doLogging = landscape["do_logging"]; + } + + auto numThreads = 1; + if (landscape.contains("num_threads")) { + numThreads = landscape["num_threads"]; + } + + config.setLocalDomain(localDomainType, iterationType, doLogging, numThreads, sequencerLibrary, sequencer, + simulateLandUnit, landUnitBuildSuccess, settings, stepping); + + config.localDomain()->setLandscapeObject(landscape["provider"].extract(), iterationType); + + auto landscapeObject = config.localDomain()->landscapeObject(); + switch (iterationType) { + case LocalDomainIterationType::AreaOfInterest: { + // NEW STUFF + landscapeObject->setIterationAreaOfInterest(); + auto aoiIteration = landscapeObject->iterationAreaOfInterest(); + // TODO: fill this in + break; + } + case LocalDomainIterationType::ASpatialIndex: { + landscapeObject->setIterationASpatialIndex(); + auto asiIteration = landscapeObject->iterationASpatialIndex(); + asiIteration->set_maxTileSize(landscape["max_tile_size"]); + asiIteration->set_tileCacheSize(landscape["tile_cache_size"]); + break; + } + case LocalDomainIterationType::ASpatialMongoIndex: { + landscapeObject->setIterationASpatialMongoIndex(); + auto asmiIteration = landscapeObject->iterationASpatialMongoIndex(); + + if (landscape.contains("process_all")) asmiIteration->set_processAlldocuments(landscape["process_all"]); + + if (landscape.contains("documents")) { + for (auto& item : landscape["documents"].extract()) { + std::string id = item["id"]; + asmiIteration->addDocumentId(id); + } + } + break; + } + case LocalDomainIterationType::LandscapeTiles: { + landscapeObject->setIterationLandscapeTiles(); + auto ltIteration = landscapeObject->iterationLandscapeTiles(); + ltIteration->set_tileSizeX(landscape["tile_size_x"]); + ltIteration->set_tileSizeY(landscape["tile_size_y"]); + ltIteration->set_xPixels(landscape["x_pixels"]); + ltIteration->set_yPixels(landscape["y_pixels"]); + for (auto& tileConfig : landscape["tiles"].extract()) { + int xIndex = tileConfig["x"]; + int yIndex = tileConfig["y"]; + ltIteration->landscapeTiles().push_back(ConfigTile(xIndex, yIndex, ltIteration->tileSizeX(), + ltIteration->tileSizeY(), ltIteration->xPixels(), + ltIteration->yPixels())); + } + break; + } + case LocalDomainIterationType::TileIndex: { + landscapeObject->setIterationTileIndex(); + auto tileIteration = landscapeObject->iterationTileIndex(); + + for (auto& item : landscape["tiles"].extract()) { + UInt32 tileIdx = item["tile_index"]; + tileIteration->addTileIndex(ConfigTileIdx(tileIdx)); + } + break; + } + case LocalDomainIterationType::BlockIndex: { + landscapeObject->setIterationBlockIndex(); + auto blockIteration = landscapeObject->iterationBlockIndex(); + for (auto& item : landscape["blocks"].extract()) { + const UInt32 tileIdx = item["tile_index"]; + const UInt32 blockIdx = item["block_index"]; + blockIteration->addBlockIndex(ConfigBlockIdx(tileIdx, blockIdx)); + } + break; + } + case LocalDomainIterationType::CellIndex: { + landscapeObject->setIterationCellIndex(); + auto cellIteration = landscapeObject->iterationCellIndex(); + for (auto& item : landscape["cells"].extract()) { + const UInt32 tileIdx = item["tile_index"]; + const UInt32 blockIdx = item["block_index"]; + const UInt32 cellIdx = item["cell_index"]; + cellIteration->addCellIndex(ConfigCellIdx(tileIdx, blockIdx, cellIdx)); + } + break; + } + case LocalDomainIterationType::NotAnIteration: + break; + default: + break; + } + } else { + config.setLocalDomain(localDomainType, LocalDomainIterationType::NotAnIteration, true, 1, sequencerLibrary, + sequencer, simulateLandUnit, landUnitBuildSuccess, settings); + } + config.localDomain()->operationManagerObject()->set_name( + operationManagerSettings["name"].extract()); + config.localDomain()->operationManagerObject()->set_settings(operationManagerSettings); +} + + +void JSON2ConfigurationProvider::createSpinup(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); + if (!jsonStruct.contains("Spinup")) { + DynamicObject settings; + config.setSpinup(false, "NA", "NA", "NA", "NA", settings); + return; + } + + auto spinupStruct = jsonStruct["Spinup"].extract(); + bool enabled = false; + if (spinupStruct.contains("enabled")) { + enabled = spinupStruct["enabled"].extract(); + } + + if (enabled) { + for (auto param : {"sequencer_library", "sequencer", "simulateLandUnit", "landUnitBuildSuccess"}) { + if (!spinupStruct.contains(param)) { + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param((boost::format("Spinup.%1%") % param).str())); + } + } + + auto sequencerLibrary = spinupStruct["sequencer_library"].extract(); + auto sequencer = spinupStruct["sequencer"].extract(); + auto simulateLandUnit = spinupStruct["simulateLandUnit"].extract(); + auto landUnitBuildSuccess = spinupStruct["landUnitBuildSuccess"].extract(); + auto d = parsePocoVarToDynamic(jsonStruct["Spinup"]); + auto& settings = d.isEmpty() ? DynamicObject() : d.extract(); + config.setSpinup(enabled, sequencerLibrary, sequencer, simulateLandUnit, landUnitBuildSuccess, settings); + } else { + DynamicObject settings; + config.setSpinup(false, "NA", "NA", "NA", "NA", settings); + } +} + + +void JSON2ConfigurationProvider::createLibraries(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); + auto librariesStruct = jsonStruct["Libraries"].extract(); + for (auto item : librariesStruct) { + if (item.second.isStruct()) { + auto libraryStruct = item.second.extract(); + auto libraryName = libraryStruct["library"].extract(); + auto libraryPath = libraryStruct["path"].extract(); // item.first; + const auto libraryTypeStr = libraryStruct["type"].extract(); + + MOJA_LOG_DEBUG << "details (" << item.first << "): path - " << libraryPath << ", filename -" << libraryName; + + config.addLibrary(libraryPath, libraryName, item.first, parseLibraryType(libraryTypeStr)); + } else { + config.addLibrary(item.first, parseLibraryType(item.second.extract())); + } + } +} + +bool JSON2ConfigurationProvider::fileExists(const std::string& path) { + MOJA_PROFILE_FUNCTION(); + Poco::File pf(path); + return pf.exists(); +} + + +void JSON2ConfigurationProvider::createProviders(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + auto jsonStruct2 = *parsedJSON.extract(); + auto provider = jsonStruct2.getObject("Providers"); + auto& data = *(provider.get()); + for (auto& item : data) { + auto d = parsePocoJSONToDynamic(item.second); + if (d.isEmpty()) { + BOOST_THROW_EXCEPTION(ProviderSettingsException() << ProviderName(item.first)); + } + auto xx = d.extract(); + std::string libName; + if (xx.contains("library")) + libName = d["library"].extract(); + else + libName = "internal.flint"; + + const std::string providerType = d["type"].extract(); + auto& settings = d.extract(); + config.addProvider(item.first, libName, providerType, settings); + } +} + + +void JSON2ConfigurationProvider::createPools(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + + Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); + auto poolsItem = jsonStruct["Pools"]; + + if (poolsItem.isStruct()) { + auto poolsStruct = poolsItem.extract(); + for (auto item : poolsStruct) { + if (item.second.isStruct()) { + auto poolStruct = item.second.extract(); + if (createSpecialPools(item.first, poolStruct, config)) { + continue; + } + + auto poolDesc = poolStruct["description"].extract(); + auto poolUnits = poolStruct["units"].extract(); + auto poolScale = poolStruct["scale"].extract(); + auto poolOrder = int(poolStruct["order"]); + config.addPool(item.first, poolDesc, poolUnits, poolScale, poolOrder, item.second.extract()); + } else { + auto v = parsePocoVarToDynamic(item.second); + config.addPool(item.first, item.second.extract()); + } + } + } else if (poolsItem.isArray()) { + auto poolsArr = poolsItem.extract(); + for (auto item : poolsArr) { + if (item.isStruct()) { + auto poolStruct = item.extract(); + if (poolStruct.size() == 1) { + auto poolName = poolStruct.begin()->first; + if (createSpecialPools(poolName, poolStruct, config)) { + continue; + } + + auto poolValue = poolStruct.begin()->second.extract(); + config.addPool(poolName, poolValue); + } else { + auto poolName = poolStruct["name"].extract(); + if (createSpecialPools(poolName, poolStruct, config)) { + continue; + } + + auto poolDesc = poolStruct["description"].extract(); + auto poolUnits = poolStruct["units"].extract(); + auto poolScale = poolStruct["scale"].extract(); + auto poolOrder = int(poolStruct["order"]); + auto poolValue = poolStruct["value"].extract(); + config.addPool(poolName, poolDesc, poolUnits, poolScale, poolOrder, poolValue); + } + } else { + // TODO: Check this, perhaps we just have a list of Pool names to set to 0? + const auto poolName = item.extract(); + config.addPool(poolName, 0.0); + } + } + } +} + +bool JSON2ConfigurationProvider::createSpecialPools(const std::string& poolName, const DynamicStruct& poolSettings, + Configuration& config) const { + MOJA_PROFILE_FUNCTION(); + if (!poolSettings.contains("transform")) { + return false; + } + + auto poolDesc = poolSettings.contains("description") ? poolSettings["description"].extract() : ""; + auto poolUnits = poolSettings.contains("units") ? poolSettings["units"].extract() : ""; + auto poolScale = poolSettings.contains("scale") ? poolSettings["scale"].extract() : 1.0; + auto poolOrder = poolSettings.contains("order") ? int(poolSettings["order"]) : 0; + + auto transformSettings = parsePocoVarToDynamic(poolSettings["transform"]); + auto settings = transformSettings.extract(); + auto typeName = transformSettings["type"].extract(); + auto libraryName = transformSettings["library"].extract(); + + config.addExternalPool(poolName, poolDesc, poolUnits, poolScale, poolOrder, libraryName, typeName, settings); + + return true; +} + +bool JSON2ConfigurationProvider::createSpecialVariables( + const std::pair& keyValPair, Configuration& config, bool isSpinup) const { + + + MOJA_PROFILE_FUNCTION(); + auto spinup = config.spinup(); + auto varName = keyValPair.first; + if (keyValPair.second.isStruct()) { + auto ss = keyValPair.second.extract(); + if (ss.size() == 1) { + auto keyValPairSS = ss.begin(); + if (keyValPairSS->first == "flintdata") { + auto v = parsePocoVarToDynamic(keyValPairSS->second); + auto varStruct = v.extract(); + auto typeName = varStruct["type"].extract(); + auto libraryName = varStruct["library"].extract(); + auto& settings = + varStruct.contains("settings") ? varStruct["settings"].extract() : DynamicObject(); + isSpinup ? spinup->addFlintDataVariable(varName, libraryName, typeName, settings) + : config.addFlintDataVariable(varName, libraryName, typeName, settings); + return true; + } + if (keyValPairSS->first == "transform") { + auto v = parsePocoVarToDynamic(keyValPairSS->second); + auto varStruct = v.extract(); + auto typeName = varStruct["type"].extract(); + auto libraryName = varStruct["library"].extract(); + isSpinup ? spinup->addExternalVariable(varName, libraryName, typeName, varStruct) + : config.addExternalVariable(varName, libraryName, typeName, varStruct); + return true; + } + } + } + return false; +} + + +void JSON2ConfigurationProvider::createVariables(Poco::Dynamic::Var& parsedJSON, Configuration& config, + bool isSpinup) const { + + MOJA_PROFILE_FUNCTION(); + auto spinup = config.spinup(); + const std::string configSection = isSpinup ? "SpinupVariables" : "Variables"; + + Poco::DynamicStruct jsonStruct2 = *parsedJSON.extract(); + auto& variablesItemVar = jsonStruct2[configSection]; + + if (variablesItemVar.isStruct()) { + const auto& variablesItem = variablesItemVar.extract(); + int index = 0; + for (const auto& keyValPair : variablesItem) { + index++; + const auto varName = keyValPair.first; + if (createSpecialVariables(keyValPair, config, isSpinup)) continue; + auto v = parsePocoVarToDynamic(keyValPair.second); + isSpinup ? spinup->addVariable(varName, v) : config.addVariable(varName, v); + } + } else if (variablesItemVar.isArray()) { + auto arr = variablesItemVar.extract(); + + int index = 0; + for (auto& value : arr) { + index++; + if (value.isStruct()) { + const auto& s = value.extract(); + if (value.size() == 1) { + auto keyValPair = s.begin(); + auto varName = keyValPair->first; + if (createSpecialVariables(*keyValPair, config, isSpinup)) continue; + auto v = parsePocoVarToDynamic(keyValPair->second); + isSpinup ? spinup->addVariable(keyValPair->first, v) : config.addVariable(keyValPair->first, v); + } else { + MOJA_LOG_ERROR << "Variable defined in array (index " << index + << ") has has struct with more than one item defined)"; + } + } else { + // TODO: do we set a default value or ignore this Variable? I'd say log error and move on + auto varName = value.extract(); + MOJA_LOG_ERROR << "Variable defined in array (index " << index + << ") has no value (not Struct or Array): " << varName; + } + } + } +} + + +void JSON2ConfigurationProvider::createModules(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); + auto modulesStruct = jsonStruct["Modules"].extract(); + for (auto item : modulesStruct) { + auto moduleStruct = item.second.extract(); + if (moduleStruct.contains("enabled")) { + const bool enabled = moduleStruct["enabled"].extract(); + if (!enabled) continue; + } + if (!moduleStruct.contains("library")) { + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); + } + if (!moduleStruct.contains("order")) { + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); + } + + const auto& moduleLibraryName = moduleStruct["library"].extract(); + auto moduleOrder = int(moduleStruct["order"]); + + bool isProxy = false; + if (moduleStruct.contains("is_proxy")) { + isProxy = moduleStruct["is_proxy"].extract(); + } + + const auto settings = parsePocoVarToDynamic(moduleStruct["settings"]); + config.addModule(moduleLibraryName, item.first, moduleOrder, isProxy, + settings.isEmpty() ? DynamicObject() : settings.extract()); + } +} + + +void JSON2ConfigurationProvider::createSpinupModules(Poco::Dynamic::Var& parsedJSON, Configuration& config) const { + + MOJA_PROFILE_FUNCTION(); + Poco::DynamicStruct jsonStruct = *parsedJSON.extract(); + auto modulesStruct = jsonStruct["SpinupModules"].extract(); + for (auto item : modulesStruct) { + auto moduleStruct = item.second.extract(); + + if (moduleStruct.contains("enabled")) { + bool enabled = moduleStruct["enabled"].extract(); + if (!enabled) continue; + } + + if (!moduleStruct.contains("library")) { + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("library")); + } + if (!moduleStruct.contains("order")) { + BOOST_THROW_EXCEPTION(ModuleParamsException() << Param("order")); + } + + const auto moduleLibraryName = moduleStruct["library"].extract(); + auto moduleOrder = int(moduleStruct["order"]); + auto moduleCreateNew = moduleStruct.contains("create_new") ? moduleStruct["create_new"].extract() : false; + + const auto settings = parsePocoVarToDynamic(moduleStruct["settings"]); + config.spinup()->addSpinupModule(moduleLibraryName, item.first, moduleOrder, moduleCreateNew, + settings.isEmpty() ? DynamicObject() : settings.extract()); + } +} + +LocalDomainType JSON2ConfigurationProvider::parseLocalDomainType(const std::string& type) { + if (type == "spatial_tiled") { + return LocalDomainType::SpatialTiled; + } else if (type == "spatially_referenced_sql") { + return LocalDomainType::SpatiallyReferencedSQL; + } else if (type == "spatially_referenced_nosql") { + return LocalDomainType::SpatiallyReferencedNoSQL; + } else if (type == "threaded_spatially_referenced_nosql") { + return LocalDomainType::ThreadedSpatiallyReferencedNoSQL; + } else if (type == "point") { + return LocalDomainType::Point; + } + throw std::runtime_error("Invalid LocalDomain Type specified"); +} + +LibraryType JSON2ConfigurationProvider::parseLibraryType(const std::string& type) { + if (type == "internal") { + return LibraryType::Internal; + } else if (type == "external") { + return LibraryType::External; + } else if (type == "managed") { + return LibraryType::Managed; + } else if (type == "python") { + return LibraryType::Python; + } + + throw std::runtime_error("TODO: specific exception here"); +} + +} // namespace configuration +} // namespace flint +} // namespace moja \ No newline at end of file diff --git a/Source/moja.flint/src/aggregatorfilewriter.cpp b/Source/moja.flint/src/aggregatorfilewriter.cpp index 0e4042a..36d7b86 100644 --- a/Source/moja.flint/src/aggregatorfilewriter.cpp +++ b/Source/moja.flint/src/aggregatorfilewriter.cpp @@ -34,7 +34,7 @@ static const auto EOL_CHR = "\r\n"; // -------------------------------------------------------------------------------------------- void AggregatorFileWriter::configure(const DynamicObject& config) { - _fileName = config["filename"].convert(); + _fileName = config["filename"].extract(); _writeFileHeader = true; if (config.contains("write_header")) { @@ -156,8 +156,8 @@ inline static std::string recordToStringFunc(DateRow& rec, const std::string& dl .str(); }; // inline static std::string recordToStringFunc(Date2Row & rec, const std::string& dl) { return -// (boost::format("%2%%1%%3%") % dl % rec.get<0>() % -// rec.get<1>()).str(); }; +// (boost::format("%2%%1%%3%") % dl % +// rec.get<0>() % rec.get<1>()).str(); }; inline static std::string recordToStringFunc(Date2Row& rec, const std::string& dl) { return (boost::format("%1%") % rec.get<1>()).str(); }; diff --git a/Source/moja.flint/src/externalpoolsimple.cpp b/Source/moja.flint/src/externalpoolsimple.cpp index 8324f53..98466dd 100644 --- a/Source/moja.flint/src/externalpoolsimple.cpp +++ b/Source/moja.flint/src/externalpoolsimple.cpp @@ -32,7 +32,7 @@ double ExternalPoolSimple::initValue() const { return _initValue->value(); } void ExternalPoolSimple::init() { const auto& value = _initValue->value(); - _value = value.isEmpty() ? 0.0 : value.convert(); + _value = value.isEmpty() ? 0.0 : value.extract(); } } // namespace flint diff --git a/Source/moja.flint/src/locationfromflintdatatransform.cpp b/Source/moja.flint/src/locationfromflintdatatransform.cpp index d980b29..043a22e 100644 --- a/Source/moja.flint/src/locationfromflintdatatransform.cpp +++ b/Source/moja.flint/src/locationfromflintdatatransform.cpp @@ -17,8 +17,8 @@ void LocationFromFlintDataTransform::configure(DynamicObject config, const ILand _landUnitController = &landUnitController; _dataRepository = &dataRepository; _provider = std::static_pointer_cast( - _dataRepository->getProvider(config["provider"].convert())); - _dataId = config["data_id"].convert(); + _dataRepository->getProvider(config["provider"].extract())); + _dataId = config["data_id"].extract(); _spatialLocationInfo = std::static_pointer_cast( landUnitController.getVariable("spatialLocationInfo")->value().extract>()); } diff --git a/Source/moja.flint/src/locationidxfromflintdatatransform.cpp b/Source/moja.flint/src/locationidxfromflintdatatransform.cpp index 7b1076f..7fd722d 100644 --- a/Source/moja.flint/src/locationidxfromflintdatatransform.cpp +++ b/Source/moja.flint/src/locationidxfromflintdatatransform.cpp @@ -16,9 +16,9 @@ void LocationIdxFromFlintDataTransform::configure(DynamicObject config, const IL datarepository::DataRepository& dataRepository) { _landUnitController = &landUnitController; auto provider = std::static_pointer_cast( - dataRepository.getProvider(config["provider"].convert())); + dataRepository.getProvider(config["provider"].extract())); _indexer = &provider->indexer(); - _layer = provider->getLayer(config["data_id"].convert()); + _layer = provider->getLayer(config["data_id"].extract()); _spatialLocationInfo = std::static_pointer_cast( landUnitController.getVariable("spatialLocationInfo")->value().extract>()); } diff --git a/Source/moja.flint/src/locationidxtransform.cpp b/Source/moja.flint/src/locationidxtransform.cpp index 51a3196..05016a5 100644 --- a/Source/moja.flint/src/locationidxtransform.cpp +++ b/Source/moja.flint/src/locationidxtransform.cpp @@ -15,9 +15,9 @@ void LocationIdxTransform::configure(DynamicObject config, const ILandUnitContro datarepository::DataRepository& dataRepository) { _landUnitController = &landUnitController; auto provider = std::static_pointer_cast( - dataRepository.getProvider(config["provider"].convert())); + dataRepository.getProvider(config["provider"].extract())); _indexer = &provider->indexer(); - _layer = provider->getLayer(config["data_id"].convert()); + _layer = provider->getLayer(config["data_id"].extract()); _tileIdx = _landUnitController->getVariable("tileIndex"); _blockIdx = _landUnitController->getVariable("blockIndex"); @@ -44,4 +44,4 @@ const DynamicVar& LocationIdxTransform::value() const { } } // namespace flint -} // namespace moja +} // namespace moja \ No newline at end of file diff --git a/Source/moja.flint/src/locationtransform.cpp b/Source/moja.flint/src/locationtransform.cpp index cce3f28..4c80c3e 100644 --- a/Source/moja.flint/src/locationtransform.cpp +++ b/Source/moja.flint/src/locationtransform.cpp @@ -16,9 +16,9 @@ void LocationTransform::configure(DynamicObject config, const ILandUnitControlle _landUnitController = &landUnitController; _dataRepository = &dataRepository; _provider = std::static_pointer_cast( - _dataRepository->getProvider(config["provider"].convert())); + _dataRepository->getProvider(config["provider"].extract())); - _dataId = config["data_id"].convert(); + _dataId = config["data_id"].extract(); _latitude = _landUnitController->getVariable("latitude"); _longitude = _landUnitController->getVariable("longitude"); } diff --git a/Source/moja.flint/src/lookuprandomtransform.cpp b/Source/moja.flint/src/lookuprandomtransform.cpp index 6415e36..37b949e 100644 --- a/Source/moja.flint/src/lookuprandomtransform.cpp +++ b/Source/moja.flint/src/lookuprandomtransform.cpp @@ -1,4 +1,3 @@ - #include "moja/flint/lookuprandomtransform.h" #include "moja/flint/flintexceptions.h" @@ -32,7 +31,7 @@ void LookupRandomTransform::configure(DynamicObject config, const ILandUnitContr } } //_fromVarName = config["from"].convert(); - _toVarName = config["to"].convert(); + _toVarName = config["to"].extract(); _reverseLookup = config.contains("reverse") ? bool(config["reverse"]) : false; _landUnitController = &landUnitController; _dataRepository = &dataRepository; diff --git a/Source/moja.flint/src/lookuptransform.cpp b/Source/moja.flint/src/lookuptransform.cpp index db06ce7..6e4d025 100644 --- a/Source/moja.flint/src/lookuptransform.cpp +++ b/Source/moja.flint/src/lookuptransform.cpp @@ -29,8 +29,8 @@ void LookupTransform::configure(DynamicObject config, const ILandUnitController& } } - _fromVarName = config["from"].convert(); - _toVarName = config["to"].convert(); + _fromVarName = config["from"].extract(); + _toVarName = config["to"].extract(); _landUnitController = &landUnitController; _dataRepository = &dataRepository; } @@ -72,10 +72,10 @@ const DynamicVar& LookupTransform::value() const { const DynamicVar& LookupTransform::matchDynamicToStruct(const DynamicVar& from, const DynamicVar& to) const { std::string key = "%1%"; if (from.isInteger()) { - int intkey = from.convert(); + int intkey = from.extract(); key = (boost::format(key) % intkey).str(); } else { - key = from.convert(); + key = from.extract(); } auto toValue = to.extract(); diff --git a/Source/moja.flint/src/proxymodule.cpp b/Source/moja.flint/src/proxymodule.cpp index 8c96266..83de9d1 100644 --- a/Source/moja.flint/src/proxymodule.cpp +++ b/Source/moja.flint/src/proxymodule.cpp @@ -100,14 +100,14 @@ void ProxyModule::onSystemInit() { } } for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::SystemInit); } } void ProxyModule::onSystemShutdown() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::SystemShutdown); } } @@ -119,119 +119,119 @@ void ProxyModule::onLocalDomainInit() { } } for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::LocalDomainInit); } } void ProxyModule::onLocalDomainShutdown() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::LocalDomainShutdown); } } void ProxyModule::onLocalDomainProcessingUnitInit() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::LocalDomainProcessingUnitInit); } } void ProxyModule::onLocalDomainProcessingUnitShutdown() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::LocalDomainProcessingUnitShutdown); } } void ProxyModule::onPreTimingSequence() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::PreTimingSequence); } } void ProxyModule::onTimingInit() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingInit); } } void ProxyModule::onTimingPostInit() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingPostInit); } } void ProxyModule::onTimingShutdown() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingShutdown); } } void ProxyModule::onTimingStep() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingStep); } } void ProxyModule::onTimingPreEndStep() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingPreEndStep); } } void ProxyModule::onTimingEndStep() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingEndStep); } } void ProxyModule::onTimingPostStep() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::TimingPostStep); } } void ProxyModule::onOutputStep() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::OutputStep); } } void ProxyModule::onError(std::string msg) { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::Error, msg); } } void ProxyModule::onDisturbanceEvent(DynamicVar n) { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::DisturbanceEvent, n); } } void ProxyModule::onPostDisturbanceEvent() { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::PostDisturbanceEvent); } } void ProxyModule::onPostNotification(short preMessageSignal) { for (auto& m : _modulesData) { - if (m._useVariable ? m._variable->value().convert() : true) + if (m._useVariable ? m._variable->value().extract() : true) m._notificationCenterPtr->postNotification(moja::signals::PostNotification, preMessageSignal); } } diff --git a/Source/moja.flint/src/sqlquerytransform.cpp b/Source/moja.flint/src/sqlquerytransform.cpp index 759a7f2..e5627ef 100644 --- a/Source/moja.flint/src/sqlquerytransform.cpp +++ b/Source/moja.flint/src/sqlquerytransform.cpp @@ -28,7 +28,7 @@ void SQLQueryTransform::configure(DynamicObject config, const ILandUnitControlle moja::datarepository::DataRepository& dataRepository) { _config = config; if (config.contains("queryString")) { - _queryStr = config["queryString"].convert(); + _queryStr = config["queryString"].extract(); } else if (config.contains("queryFile")) { _queryStr = readSQLFile(config["queryFile"]); } @@ -202,7 +202,7 @@ std::string SQLQueryTransform::formatVariableValues(const IVariable& var, Dynami } } for (auto value : values) { - auto str = value.convert(); + auto str = value.extract(); if (value.isString()) { // Enclose string values in SQL single quotes. str = "'" + str + "'"; @@ -255,4 +255,4 @@ void SQLQueryTransform::replaceTokens(const std::string& tokenType, std::string& } } // namespace flint -} // namespace moja +} // namespace moja \ No newline at end of file diff --git a/Source/moja.flint/src/writesystemconfig.cpp b/Source/moja.flint/src/writesystemconfig.cpp index 849e727..546d411 100644 --- a/Source/moja.flint/src/writesystemconfig.cpp +++ b/Source/moja.flint/src/writesystemconfig.cpp @@ -1,12 +1,12 @@ +#include "moja/flint/writesystemconfig.h" + +#include "moja/flint/externalvariable.h" +#include "moja/flint/flintdatavariable.h" #include "moja/flint/ilandunitdatawrapper.h" #include "moja/flint/ipool.h" #include "moja/flint/itiming.h" #include "moja/flint/spatiallocationinfo.h" - -#include "moja/flint/externalvariable.h" -#include "moja/flint/flintdatavariable.h" #include "moja/flint/variable.h" -#include "moja/flint/writesystemconfig.h" #include #include @@ -24,8 +24,13 @@ #include #include +//Boost Format #include + +// FMT Format +#include + #include #include @@ -35,11 +40,11 @@ namespace flint { // -------------------------------------------------------------------------------------------- void WriteSystemConfig::configure(const DynamicObject& config) { - _name = config["name"].convert(); - _outputPath = config.contains("output_path") ? config["output_path"].convert() : ""; - _writeFrequency = config.contains("write_frequency") ? config["write_frequency"].convert() : 0; + _name = config["name"].extract(); + _outputPath = config.contains("output_path") ? config["output_path"].extract() : ""; + _writeFrequency = config.contains("write_frequency") ? config["write_frequency"].extract() : 0; _writeOutstepFrequency = - config.contains("write_outstep_frequency") ? config["write_outstep_frequency"].convert() : 0; + config.contains("write_outstep_frequency") ? config["write_outstep_frequency"].extract() : 0; // set all notifications to false for (auto& val : _onNotificationArray) val = false; @@ -49,11 +54,11 @@ void WriteSystemConfig::configure(const DynamicObject& config) { if (dynamic.isVector()) { const auto& arr = dynamic.extract(); for (auto& val : arr) { - auto notStr = val.convert(); + auto notStr = val.extract(); _onNotificationArray[convertNotificationStringToIndex(notStr)] = true; } } else { - _onNotificationArray[convertNotificationStringToIndex(config["on_notification"].convert())] = + _onNotificationArray[convertNotificationStringToIndex(config["on_notification"].extract())] = true; } } else { @@ -87,7 +92,7 @@ void WriteSystemConfig::onSystemInit() { if (workingFolder.exists() && !workingFolder.isDirectory()) { MOJA_LOG_ERROR << "Error creating spatial tiled point configurations output folder: " << _outputPath; } - auto outputFolderPath = (boost::format("%1%%2%%3%") % workingFolder.path() % Poco::Path::separator() % _name).str(); + auto outputFolderPath = (fmt::format("{0}{1}{2}", workingFolder.path(), Poco::Path::separator(), _name)); Poco::File outputFolder(outputFolderPath); if (!outputFolder.exists()) { try { @@ -256,7 +261,7 @@ void outputDynamicToStream(std::ofstream& fout, const DynamicVar& object, int le if (object.type() == typeid(DateTime)) { DateTime dt = object.extract(); std::string simpleDateStr = - (boost::format("{ \"$date\": \"%1%/%2%/%3%\" }") % dt.year() % dt.month() % dt.day()).str(); + (fmt::format("{ \"$date\": \"{0}/{1}/{2}\" }" , dt.year() , dt.month() , dt.day())); fout << simpleDateStr; // fout << "\"" << escape_json(simpleDateStr) << "\""; } else if (object.type() == typeid(Int16)) { @@ -353,15 +358,14 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { DynamicObject localdomain; auto localDomainConfig = config->localDomain(); localdomain["type"] = configuration::LocalDomain::localDomainTypeToStr(localDomainConfig->type()); - localdomain["start_date_init"] = (boost::format("%1%/%2%/%3%") % config->startDate().year() % - config->startDate().month() % config->startDate().day()) - .str(); - localdomain["start_date"] = (boost::format("%1%/%2%/%3%") % timing->curStartDate().year() % - timing->curStartDate().month() % timing->curStartDate().day()) - .str(); + localdomain["start_date_init"] = (fmt::format("{0}/{1}/{2}", config->startDate().year() , + config->startDate().month() , config->startDate().day())); + + localdomain["start_date"] = (fmt::format("{0}/{1}/{2}" , timing->curStartDate().year() , + timing->curStartDate().month() , timing->curStartDate().day())); localdomain["end_date"] = - (boost::format("%1%/%2%/%3%") % config->endDate().year() % config->endDate().month() % config->endDate().day()) - .str(); + (fmt::format("{0}/{1}/{2}" , config->endDate().year() , config->endDate().month() , config->endDate().day())); + localdomain["sequencer_library"] = localDomainConfig->sequencerLibrary(); localdomain["sequencer"] = localDomainConfig->sequencer(); localdomain["simulateLandUnit"] = localDomainConfig->simulateLandUnit(); @@ -387,7 +391,7 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { case configuration::LocalDomainIterationType::CellIndex: { localdomain["landscape"] = DynamicObject( {//{ "iteration_type", - //configuration::convertLocalDomainIterationTypeToStr(localDomainConfig->landscapeObject()->iterationType()) + // configuration::convertLocalDomainIterationTypeToStr(localDomainConfig->landscapeObject()->iterationType()) //}, {"iteration_type", "CellIndex"}, {"num_threads", localDomainConfig->numThreads()}, @@ -408,7 +412,9 @@ void WriteSystemConfig::WriteConfig(std::string notificationStr) const { } break; case configuration::LocalDomainType::Point: { } break; - default: { } break; } + default: { + } break; + } // Build pools DynamicObject pools; diff --git a/Source/moja.flint/src/writevariablegrid.cpp b/Source/moja.flint/src/writevariablegrid.cpp index b9289f9..9284f2f 100644 --- a/Source/moja.flint/src/writevariablegrid.cpp +++ b/Source/moja.flint/src/writevariablegrid.cpp @@ -25,7 +25,7 @@ namespace moja { namespace flint { void WriteVariableGrid::configure(const DynamicObject& config) { - _globalOutputPath = config.contains("output_path") ? config["output_path"].convert() : ""; + _globalOutputPath = config.contains("output_path") ? config["output_path"].extract() : ""; _useIndexesForFolderName = false; if (config.contains("use_indexes_for_folder_name")) { @@ -47,7 +47,7 @@ void WriteVariableGrid::configure(const DynamicObject& config) { auto enabled = true; if (itemConfig.contains("enabled")) enabled = itemConfig["enabled"]; if (enabled) { - const auto variableDataType = itemConfig["variable_data_type"].convert(); + const auto variableDataType = itemConfig["variable_data_type"].extract(); if (variableDataType == "UInt8") { _dataVecT.emplace_back(std::make_unique>(_fileHandlingMutex, 1));