add priority and skip lists

Author: rinigus

importer/data/prioritylist importer/data/priority.listimporter/data/prioritylist renamed to importer/data/priority.list

@@ -1,6 +1,7 @@
 #include "hierarchyitem.h"
 #include "utils.h"
 
+#include <fstream>
 #include <iostream>
 #include <stdexcept>
 
@@ -36,6 +37,60 @@ HierarchyItem::HierarchyItem(const pqxx::row &row)
     m_name_extra = get_with_def(m_data_extra, "brand");
 }
 
+// trim from start (in place)
+static inline void ltrim(std::string &s)
+{
+  s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](int ch) { return !std::isspace(ch); }));
+}
+
+// trim from end (in place)
+static inline void rtrim(std::string &s)
+{
+  s.erase(std::find_if(s.rbegin(), s.rend(), [](int ch) { return !std::isspace(ch); }).base(),
+          s.end());
+}
+
+// trim from both ends (in place)
+static inline void trim(std::string &s)
+{
+  ltrim(s);
+  rtrim(s);
+}
+
+static std::set<std::string> load_list(const std::string &fname)
+{
+  std::set<std::string> d;
+  if (fname.empty())
+    return d;
+
+  std::ifstream f(fname);
+  std::string   line;
+  if (!f)
+    {
+      std::cerr << "Failed to open a file: " << fname << std::endl;
+      throw std::runtime_error("File cannot be opened");
+    }
+
+  while (std::getline(f, line))
+    {
+      trim(line);
+      if (!line.empty())
+        d.insert(line);
+    }
+
+  return d;
+}
+
+void HierarchyItem::load_priority_list(const std::string &fname)
+{
+  s_priority_types = load_list(fname);
+}
+
+void HierarchyItem::load_skip_list(const std::string &fname)
+{
+  s_skip_types = load_list(fname);
+}
+
 bool HierarchyItem::keep() const
 {
   if (m_type.find_first_not_of(allowed_type_chars) != std::string::npos)

importer/src/hierarchyitem.cpp

@@ -33,6 +33,10 @@ class HierarchyItem
 
   void print_branch(unsigned int offset) const;
 
+public:
+  static void load_priority_list(const std::string &fname);
+  static void load_skip_list(const std::string &fname);
+
 private:
   hindex m_id;
   hindex m_linked_id{ 0 };

importer/src/hierarchyitem.h

@@ -35,18 +35,29 @@ int main(int argc, char *argv[])
   std::string database_path;
   std::string postal_country_parser;
   std::string postal_address_parser_dir;
+  std::string type_priority_list;
+  std::string type_skip_list;
   bool        verbose_address_expansion = false;
 
   {
     po::options_description generic("Geocoder NLP importer options");
-    generic.add_options()("help", "Help message")("version,v", "Data format version")(
-        "poly,p", po::value<std::string>(&polyjson),
-        "Boundary of the imported region in GeoJSON format")(
-        "postal-country", po::value<std::string>(&postal_country_parser),
-        "libpostal country preference for this database")(
+    generic.add_options()("help", "Help message")("version,v", "Data format version");
+    generic.add_options()("poly,p", po::value<std::string>(&polyjson),
+                          "Boundary of the imported region in GeoJSON format");
+    generic.add_options()("postal-country", po::value<std::string>(&postal_country_parser),
+                          "libpostal country preference for this database");
+    generic.add_options()(
         "postal-address", po::value<std::string>(&postal_address_parser_dir),
         "libpostal address parser directory. If not specified, global libpostal parser directory "
-        "preference is used.")("verbose", "Verbose address expansion");
+        "preference is used.");
+    generic.add_options()(
+        "priority", po::value<std::string>(&type_priority_list),
+        "File with OSM tags that are kept even if there is no name associated with the location");
+    generic.add_options()(
+        "skip", po::value<std::string>(&type_skip_list),
+        "File with OSM tags for locations that should be dropped even if there is a name "
+        "associated with the location");
+    generic.add_options()("verbose", "Verbose address expansion");
 
     po::options_description hidden("Hidden options");
     hidden.add_options()("output-directory", po::value<std::string>(&database_path),
@@ -103,6 +114,9 @@ int main(int argc, char *argv[])
       std::cout << "Loaded border GeoJSON. Geometry string length: " << border.size() << "\n";
     }
 
+  HierarchyItem::load_priority_list(type_priority_list);
+  HierarchyItem::load_skip_list(type_skip_list);
+
   Hierarchy hierarchy;
 
   std::string postgres_dblink;