From fc1effe6c285fce15afb5457270ee220d3d09013 Mon Sep 17 00:00:00 2001 From: Alex Ormenisan Date: Mon, 30 May 2022 14:51:15 +0200 Subject: [PATCH 1/2] remove _type from documents and bulk request --- include/ElasticSearchBase.h | 1 - src/ElasticSearchBase.cpp | 4 ++-- src/FileProvenanceElasticDataReader.cpp | 9 --------- 3 files changed, 2 insertions(+), 12 deletions(-) diff --git a/include/ElasticSearchBase.h b/include/ElasticSearchBase.h index e1fe927b..7f79eb60 100644 --- a/include/ElasticSearchBase.h +++ b/include/ElasticSearchBase.h @@ -42,6 +42,5 @@ class ElasticSearchBase : public TimedRestBatcher, public const bool mStats; MovingCountersSet* const mCounters; - const std::string DEFAULT_TYPE; }; #endif //EPIPE_ELASTICSEARCHBASE_H diff --git a/src/ElasticSearchBase.cpp b/src/ElasticSearchBase.cpp index 74c16e16..34160f73 100644 --- a/src/ElasticSearchBase.cpp +++ b/src/ElasticSearchBase.cpp @@ -24,11 +24,11 @@ elastic_client_config, int time_to_wait_before_inserting, int bulk_size, const bool statsEnabled, MovingCountersSet* const metricsCounters) : TimedRestBatcher(elastic_client_config, time_to_wait_before_inserting, bulk_size), mStats - (statsEnabled), mCounters(metricsCounters), DEFAULT_TYPE("_doc") { + (statsEnabled), mCounters(metricsCounters) { } std::string ElasticSearchBase::getElasticSearchBulkUrl(std::string index) { - std::string str = "/" + index + "/" + DEFAULT_TYPE + "/_bulk"; + std::string str = "/" + index + "/_bulk"; return str; } diff --git a/src/FileProvenanceElasticDataReader.cpp b/src/FileProvenanceElasticDataReader.cpp index 645d9df0..7a818d12 100644 --- a/src/FileProvenanceElasticDataReader.cpp +++ b/src/FileProvenanceElasticDataReader.cpp @@ -31,7 +31,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("update", opVal, opAlloc); @@ -80,7 +79,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("update", opVal, opAlloc); @@ -117,7 +115,6 @@ class ElasticHelper { rapidjson::Value cleanupIdVal(rapidjson::kObjectType); cleanupIdVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), cleanupAlloc), cleanupAlloc); - cleanupIdVal.AddMember("_type", rapidjson::Value().SetString("_doc", cleanupAlloc), cleanupAlloc); cleanupIdVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), cleanupAlloc), cleanupAlloc); cleanupId.AddMember("update", cleanupIdVal, cleanupId.GetAllocator()); @@ -147,7 +144,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("update", opVal, opAlloc); @@ -210,7 +206,6 @@ class ElasticHelper { rapidjson::Value cleanupIdVal(rapidjson::kObjectType); cleanupIdVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), cleanupAlloc), cleanupAlloc); - cleanupIdVal.AddMember("_type", rapidjson::Value().SetString("_doc", cleanupAlloc), cleanupAlloc); cleanupIdVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), cleanupAlloc), cleanupAlloc); cleanupId.AddMember("update", cleanupIdVal, cleanupAlloc); @@ -253,7 +248,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("delete", opVal, opAlloc); @@ -275,7 +269,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("update", opVal, opAlloc); @@ -327,7 +320,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("update", opVal, opAlloc); @@ -392,7 +384,6 @@ class ElasticHelper { rapidjson::Value opVal(rapidjson::kObjectType); opVal.AddMember("_id", rapidjson::Value().SetString(id.c_str(), opAlloc), opAlloc); - opVal.AddMember("_type", rapidjson::Value().SetString("_doc", opAlloc), opAlloc); opVal.AddMember("_index", rapidjson::Value().SetString(index.c_str(), opAlloc), opAlloc); op.AddMember("update", opVal, opAlloc); From 030cf184a89ebba22df506f202a05c875b0b1263 Mon Sep 17 00:00:00 2001 From: Alex Ormenisan Date: Mon, 30 May 2022 16:56:08 +0200 Subject: [PATCH 2/2] update boost download location --- docker/centos/Dockerfile | 2 +- docker/ubuntu/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/centos/Dockerfile b/docker/centos/Dockerfile index 795b3976..5d8cac79 100644 --- a/docker/centos/Dockerfile +++ b/docker/centos/Dockerfile @@ -42,7 +42,7 @@ rm -r cmake-${cmake_version}* RUN echo Installing Boost; \ BOOST_V=`echo ${boost_version} | sed s/'\.'/'_'/g`; \ source scl_source enable devtoolset-8; \ -wget --progress=bar:force https://dl.bintray.com/boostorg/release/${boost_version}/source/boost_${BOOST_V}.tar.gz; \ +wget --progress=bar:force https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${BOOST_V}.tar.gz; \ tar xzf boost_${BOOST_V}.tar.gz; \ cd boost_${BOOST_V}; \ ./bootstrap.sh; \ diff --git a/docker/ubuntu/Dockerfile b/docker/ubuntu/Dockerfile index 592a40d1..8f68ea5c 100644 --- a/docker/ubuntu/Dockerfile +++ b/docker/ubuntu/Dockerfile @@ -28,7 +28,7 @@ rm -r cmake-${cmake_version}* RUN echo Installing Boost; \ BOOST_V=`echo ${boost_version} | sed s/'\.'/'_'/g`; \ -wget --progress=bar:force https://dl.bintray.com/boostorg/release/${boost_version}/source/boost_${BOOST_V}.tar.gz; \ +wget --progress=bar:force https://boostorg.jfrog.io/artifactory/main/release/${boost_version}/source/boost_${BOOST_V}.tar.gz; \ tar xzf boost_${BOOST_V}.tar.gz; \ cd boost_${BOOST_V}; \ ./bootstrap.sh; \