Add import cleanup

Author: rinigus

importer/src/hierarchy.cpp

@@ -55,6 +55,27 @@ void Hierarchy::add_linked_item(std::shared_ptr<HierarchyItem> &item)
   tolink->second->add_linked(item);
 }
 
+void Hierarchy::cleanup()
+{
+  for (auto root_iter = m_root.begin(); root_iter != m_root.end(); ++root_iter)
+    {
+      std::set<std::shared_ptr<HierarchyItem> > keep;
+      for (auto item : root_iter->second)
+        {
+          item->cleanup_children();
+          if (item->keep())
+            keep.insert(item);
+          else
+            keep.insert(item->children().begin(), item->children().end());
+        }
+      root_iter->second = keep;
+
+      // ensure that the parent is set correctly
+      for (auto item : root_iter->second)
+        item->set_parent(root_iter->first, true);
+    }
+}
+
 void Hierarchy::set_country(const std::string &country, hindex id)
 {
   if (!m_items.count(id))
@@ -85,6 +106,9 @@ void Hierarchy::finalize()
       index = item->index(index, 0);
       item->set_parent(0);
     }
+
+  std::cout << "Hierarchy: active items: " << index
+            << " / cleared items: " << m_items.size() - index << "\n";
 }
 
 void Hierarchy::write(sqlite3pp::database &db) const

importer/src/hierarchy.h

@@ -21,6 +21,7 @@ class Hierarchy
   void add_item(std::shared_ptr<HierarchyItem> &item);
   void add_linked_item(std::shared_ptr<HierarchyItem> &item);
   void set_country(const std::string &country, hindex id);
+  void cleanup();
   void finalize();
   void write(sqlite3pp::database &db) const;
 

importer/src/hierarchyitem.cpp

@@ -4,54 +4,100 @@
 #include <iostream>
 #include <stdexcept>
 
+std::set<std::string> HierarchyItem::s_priority_types;
+std::set<std::string> HierarchyItem::s_skip_types;
+
+static std::string allowed_type_chars = "abcdefghijklmnopqrstuvwxyz_-";
+
 HierarchyItem::HierarchyItem(const pqxx::row &row)
 {
   m_id          = row["place_id"].as<hindex>(0);
   m_linked_id   = row["linked_place_id"].as<hindex>(0);
   m_parent_id   = row["parent_place_id"].as<hindex>(0);
   m_country     = row["country_code"].as<std::string>("");
-  m_class       = row["class"].as<std::string>("");
-  m_type        = row["type"].as<std::string>("");
+  m_type        = geocoder_type(row["class"].as<std::string>(""), row["type"].as<std::string>(""));
   m_housenumber = row["housenumber"].as<std::string>("");
   m_postcode    = row["postcode"].as<std::string>("");
   m_latitude    = row["latitude"].as<float>(0);
   m_longitude   = row["longitude"].as<float>(0);
 
-  m_name  = parse_to_map(row["name"].as<std::string>(""));
-  m_extra = parse_to_map(row["extra"].as<std::string>(""));
+  m_data_name  = parse_to_map(row["name"].as<std::string>(""));
+  m_data_extra = parse_to_map(row["extra"].as<std::string>(""));
+
+  m_name = get_with_def(m_data_name, "name");
+  m_name_extra.clear();
+  if (!m_housenumber.empty())
+    {
+      m_name_extra = m_name;
+      m_name       = m_housenumber;
+    }
+
+  if (m_name_extra.empty())
+    m_name_extra = get_with_def(m_data_extra, "brand");
+}
+
+bool HierarchyItem::keep() const
+{
+  if (m_type.find_first_not_of(allowed_type_chars) != std::string::npos)
+    {
+      std::cout << "Dropping " << m_type << "\n";
+      return false;
+    }
+  if (s_skip_types.count(m_type) > 0)
+    return false;
+  return !m_name.empty() || s_priority_types.count(m_type) > 0;
 }
 
 void HierarchyItem::add_child(std::shared_ptr<HierarchyItem> child)
 {
   m_children.push_back(child);
+  child->set_parent(m_id);
 }
 
 void HierarchyItem::add_linked(std::shared_ptr<HierarchyItem> linked)
 {
-  m_name.insert(linked->m_name.begin(), linked->m_name.end());
-  m_extra.insert(linked->m_extra.begin(), linked->m_extra.end());
-  //   if (m_class != linked->m_class || m_type != linked->m_type)
-  //     std::cout << "Mismatch between linked objects: " << m_id << ", " << m_class << ", " <<
-  //     m_type
-  //               << " -> " << linked->m_id << ", " << linked->m_class << ", " << linked->m_type
-  //               << "\n";
+  m_data_name.insert(linked->m_data_name.begin(), linked->m_data_name.end());
+  m_data_extra.insert(linked->m_data_extra.begin(), linked->m_data_extra.end());
 }
 
-void HierarchyItem::set_parent(hindex parent)
+void HierarchyItem::set_parent(hindex parent, bool force)
 {
-  if (m_parent_id != parent && m_parent_id != 0)
+  if (!force && m_parent_id != parent && m_parent_id != 0 && parent != 0)
     {
       std::cout << "New parent (" << parent << ") for " << m_id << " does not match old one ("
                 << m_parent_id << ")\n";
       throw std::runtime_error("Mismatch between new and old parent");
     }
   m_parent_id = parent;
-  for (auto c : m_children)
-    c->set_parent(m_id);
+  //   for (auto c : m_children)
+  //     c->set_parent(m_id, force);
+}
+
+void HierarchyItem::cleanup_children()
+{
+  // as a result of this run, children that are supposed to be kept are staying in children
+  // property. all disposed ones are still pointed to via Hierarchy map, but should not be accessed
+  // while moving along hierarchy for indexing or writing it
+  std::deque<std::shared_ptr<HierarchyItem> > children;
+  for (auto item : m_children)
+    {
+      item->cleanup_children();
+      if (item->keep())
+        children.push_back(item);
+      else
+        children.insert(children.end(), item->m_children.begin(), item->m_children.end());
+    }
+  m_children = children;
+
+  // set parent, forced
+  for (auto item : m_children)
+    item->set_parent(m_id, true);
 }
 
 sqlid HierarchyItem::index(sqlid idx, sqlid parent)
 {
+  if (!keep())
+    throw std::runtime_error("Trying to index a location that was not supposed to be kept");
   m_my_index     = idx;
   m_parent_index = parent;
   ++idx;
@@ -63,27 +109,20 @@ sqlid HierarchyItem::index(sqlid idx, sqlid parent)
 
 void HierarchyItem::write(sqlite3pp::database &db) const
 {
-  // primary data
-  std::string name    = get_with_def(m_name, "name");
-  std::string name_en = get_with_def(m_name, "name:en");
-  std::string phone   = get_with_def(m_extra, "phone");
-  std::string website = get_with_def(m_extra, "website");
+  if (!keep())
+    throw std::runtime_error("Trying to write a location that was not supposed to be kept");
 
-  std::string name_extra;
-  if (!m_housenumber.empty())
-    {
-      name_extra = name;
-      name       = m_housenumber;
-    }
+  // primary data
+  std::string name_en = get_with_def(m_data_name, "name:en");
+  std::string phone   = get_with_def(m_data_extra, "phone");
+  std::string website = get_with_def(m_data_extra, "website");
 
-  if (name_extra.empty())
-    name_extra = get_with_def(m_extra, "brand");
   {
     sqlite3pp::command cmd(db, "INSERT INTO object_primary_tmp (id, postgres_id, name, name_extra, "
                                "name_en, phone, postal_code, website, parent, longitude, "
                                "latitude) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)");
-    cmd.binder() << m_my_index << (int)m_id << name << name_extra << name_en << phone << m_postcode
-                 << website << m_parent_index << m_longitude << m_latitude;
+    cmd.binder() << m_my_index << (int)m_id << m_name << m_name_extra << name_en << phone
+                 << m_postcode << website << m_parent_index << m_longitude << m_latitude;
     if (cmd.execute() != SQLITE_OK)
       std::cerr << "WriteSQL : error inserting primary data for " << m_id << ", " << m_my_index
                 << "\n";
@@ -95,7 +134,7 @@ void HierarchyItem::write(sqlite3pp::database &db) const
         //= "INSERT INTO object_type_tmp (prim_id, type) VALUES (?, \"" + type + "\")";
         = "INSERT INTO object_type_tmp (prim_id, type) VALUES (?, ?)";
     sqlite3pp::command cmd(db, command.c_str());
-    cmd.binder() << m_my_index << geocoder_type(m_class, m_type);
+    cmd.binder() << m_my_index << m_type;
     if (cmd.execute() != SQLITE_OK)
       std::cerr << "WriteSQL: error inserting type for " << m_id << ", " << m_my_index << "\n";
   }
@@ -120,7 +159,7 @@ void HierarchyItem::print_branch(unsigned int offset) const
   std::cout << std::string(offset, ' ') << "- " << m_id << " ";
   if (!m_housenumber.empty())
     std::cout << "house " << m_housenumber << " ";
-  for (const auto &i : m_name)
+  for (const auto &i : m_data_name)
     std::cout << i.first << ": " << i.second << " ";
   std::cout << "(" << m_my_index << " " << m_last_child_index - m_my_index << ": " << m_parent_id
             << ", " << m_country << ")\n";

importer/src/hierarchyitem.h

@@ -6,6 +6,7 @@
 #include <deque>
 #include <memory>
 #include <pqxx/pqxx>
+#include <set>
 #include <sqlite3pp.h>
 #include <string>
 
@@ -19,10 +20,14 @@ class HierarchyItem
   hindex             linked_id() const { return m_linked_id; }
   hindex             parent_id() const { return m_parent_id; }
   const std::string &country() const { return m_country; }
+  bool               keep() const;
+
+  const std::deque<std::shared_ptr<HierarchyItem> > &children() { return m_children; }
 
   void  add_child(std::shared_ptr<HierarchyItem> child);
   void  add_linked(std::shared_ptr<HierarchyItem> linked);
-  void  set_parent(hindex);
+  void  set_parent(hindex parent, bool force = false);
+  void  cleanup_children();
   sqlid index(sqlid idx, sqlid parent);
   void  write(sqlite3pp::database &db) const;
 
@@ -36,18 +41,22 @@ class HierarchyItem
   sqlid  m_parent_index;
   sqlid  m_last_child_index;
 
-  std::string m_class;
   std::string m_type;
   float       m_latitude;
   float       m_longitude;
   std::string m_country;
   std::string m_postcode;
   std::string m_housenumber;
+  std::string m_name;
+  std::string m_name_extra;
 
-  std::map<std::string, std::string> m_name;
-  std::map<std::string, std::string> m_extra;
+  std::map<std::string, std::string> m_data_name;
+  std::map<std::string, std::string> m_data_extra;
 
   std::deque<std::shared_ptr<HierarchyItem> > m_children;
+
+  static std::set<std::string> s_priority_types;
+  static std::set<std::string> s_skip_types;
 };
 
 #endif

importer/src/main.cpp

@@ -143,23 +143,35 @@ int main(int argc, char *argv[])
       }
   }
 
+  // find missing parents for root nodes
   std::cout << "Fill missing hierarchies. Root size: " << hierarchy.get_root_count() << "\n";
   for (hindex parent = hierarchy.get_next_nonzero_root_parent(); parent;)
     {
-      pqxx::result r = txn.exec_params(base_query + "where place_id=$1", parent);
+      pqxx::result r     = txn.exec_params(base_query + "where place_id=$1", parent);
+      bool         found = false;
       for (auto row : r)
         {
           std::shared_ptr<HierarchyItem> item = std::make_shared<HierarchyItem>(row);
           hierarchy.add_item(item);
+          found = true;
+        }
+
+      if (!found)
+        {
+          std::cerr << "Missing parent with ID " << parent << ". Stopping import\n";
+          return -1;
         }
 
       parent = hierarchy.get_next_nonzero_root_parent();
     }
 
-  std::cout << "Try to fill missing parents through countries. Root size: "
-            << hierarchy.get_root_count() << "\n";
+  // remove all items from hierarchy that are not supposed to be there
+  std::cout << "Cleanup hierarchy\n";
+  hierarchy.cleanup();
 
   // find missing countries and move root nodes under them if possible
+  std::cout << "Try to fill missing parents through countries. Root size: "
+            << hierarchy.get_root_count() << "\n";
   for (std::string country : hierarchy.get_root_countries())
     {
       for (auto row : txn.exec_params(
@@ -171,7 +183,7 @@ int main(int argc, char *argv[])
 
   txn.commit(); // finalize postgres transactions
 
-  hierarchy.print(false);
+  // hierarchy.print(false);
 
   // Saving data into SQLite
   sqlite3pp::database db(GeoNLP::Geocoder::name_primary(database_path).c_str());
@@ -261,6 +273,22 @@ int main(int argc, char *argv[])
              "SELECT box_id, min(latitude), max(latitude), min(longitude), max(longitude) from "
              "object_primary group by box_id");
 
+  // Stats view
+  db.execute("DROP VIEW IF EXISTS type_stats");
+  db.execute(
+      "CREATE VIEW type_stats AS SELECT t.name as type_name, COUNT(*) AS cnt FROM object_primary o "
+      "JOIN \"type\" t ON t.id = o.type_id GROUP BY t.name ORDER BY cnt desc");
+  {
+    std::cout << "List of most popular imported types\n";
+    sqlite3pp::query qry(db, "SELECT type_name, cnt FROM type_stats ORDER BY cnt DESC LIMIT 25");
+    for (auto v : qry)
+      {
+        std::string name;
+        int         cnt;
+        v.getter() >> name >> cnt;
+        std::cout << " " << name << "\t" << cnt << "\n";
+      }
+  }
   // Recording version
   db.execute("DROP TABLE IF EXISTS meta");
   db.execute("CREATE TABLE meta (key TEXT, value TEXT)");

importer/src/utils.cpp

@@ -27,7 +27,7 @@ std::map<std::string, std::string> parse_to_map(const std::string &js)
 
 std::string geocoder_type(const std::string &t_class, const std::string &t_value)
 {
-  if (t_value == "yes")
+  if (t_value == "yes" || t_value.empty())
     return t_class;
   return t_class + "_" + t_value;
 }