Merge branch 'dev' into cholmcc_aod_producers_refactor

Author: cholmcc

Algorithm/include/Algorithm/PageParser.h

@@ -255,12 +255,12 @@ class PageParser
       return mElement;
     }
     // comparison
-    bool operator==(const SelfType& rh)
+    bool operator==(const SelfType& rh) const
     {
       return mPosition == rh.mPosition;
     }
     // comparison
-    bool operator!=(const SelfType& rh)
+    bool operator!=(const SelfType& rh) const
     {
       return mPosition != rh.mPosition;
     }

CCDB/include/CCDB/CCDBDownloader.h

@@ -51,6 +51,7 @@ typedef struct DownloaderRequestData {
   long timestamp;
   HeaderObjectPair_t hoPair;
   std::map<std::string, std::string>* headers;
+  std::string userAgent;
 
   std::function<bool(std::string)> localContentCallback;
   bool errorflag = false;
@@ -208,12 +209,17 @@ class CCDBDownloader
    */
   void runLoop(bool noWait);
 
- private:
+  /**
+   * Returns a message describing the transfer an it's result.
+   */
+  std::string prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map<std::string, std::string>* headers, long httpCode) const;
+
   /**
    * Leaves only the protocol and host part of the url, discrading path and metadata.
    */
   std::string trimHostUrl(std::string full_host_url) const;
 
+ private:
   /**
    * Recognizes whether the address is a full url, or a partial one (like for example "/Task/Detector/1") and combines it with potentialHost if needed.
    */

CCDB/include/CCDB/CcdbApi.h

@@ -373,7 +373,7 @@ class CcdbApi //: public DatabaseInterface
   void scheduleDownload(RequestContext& requestContext, size_t* requestCounter) const;
 
   void getFromSnapshot(bool createSnapshot, std::string const& path,
-                       long timestamp, std::map<std::string, std::string> headers,
+                       long timestamp, std::map<std::string, std::string>& headers,
                        std::string& snapshotpath, o2::pmr::vector<char>& dest, int& fromSnapshot, std::string const& etag) const;
   void releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, std::string path) const;
   boost::interprocess::named_semaphore* createNamedSempahore(std::string path) const;

CCDB/src/CCDBDownloader.cxx

@@ -408,7 +408,7 @@ std::string CCDBDownloader::trimHostUrl(std::string full_host_url) const
   char* host;
   CURLUcode host_result = curl_url_get(host_url, CURLUPART_HOST, &host, 0);
   if (host_result != CURLUE_OK) {
-    LOG(error) << "CCDBDownloader: Malformed url detected when processing redirect, could not identify the host part: " << host;
+    LOG(error) << "CCDBDownloader: Malformed url detected when processing redirect, could not identify the host part: " << full_host_url;
     curl_url_cleanup(host_url);
     return "";
   }
@@ -493,21 +493,25 @@ void CCDBDownloader::transferFinished(CURL* easy_handle, CURLcode curlCode)
       curl_easy_getinfo(easy_handle, CURLINFO_EFFECTIVE_URL, &url);
       LOG(debug) << "Transfer for " << url << " finished with code " << httpCode << "\n";
 
+      std::string currentHost = requestData->hosts[performData->hostInd];
+      std::string loggingMessage = prepareLogMessage(currentHost, requestData->userAgent, requestData->path, requestData->timestamp, requestData->headers, httpCode);
+
       // Get alternative locations for the same host
       auto locations = getLocations(&(requestData->hoPair.header));
 
       // React to received http code
-      if (404 == httpCode) {
-        LOG(error) << "Requested resource does not exist: " << url;
-      } else if (304 == httpCode) {
-        LOGP(debug, "Object exists but I am not serving it since it's already in your possession");
-        contentRetrieved = true;
-      } else if (300 <= httpCode && httpCode < 400 && performData->locInd < locations.size()) {
-        followRedirect(performData, easy_handle, locations, rescheduled, contentRetrieved);
-      } else if (200 <= httpCode && httpCode < 300) {
-        contentRetrieved = true;
+      if (200 <= httpCode && httpCode < 400) {
+        LOG(debug) << loggingMessage;
+        if (304 == httpCode) {
+          LOGP(debug, "Object exists but I am not serving it since it's already in your possession");
+          contentRetrieved = true;
+        } else if (300 <= httpCode && httpCode < 400 && performData->locInd < locations.size()) {
+          followRedirect(performData, easy_handle, locations, rescheduled, contentRetrieved);
+        } else if (200 <= httpCode && httpCode < 300) {
+          contentRetrieved = true;
+        }
       } else {
-        LOG(error) << "Error in fetching object " << url << ", curl response code:" << httpCode;
+        LOG(error) << loggingMessage;
       }
 
       // Check if content was retrieved, or scheduled to be retrieved
@@ -694,4 +698,21 @@ void CCDBDownloader::asynchSchedule(CURL* handle, size_t* requestCounter)
   // return codeVector;
 }
 
+std::string CCDBDownloader::prepareLogMessage(std::string host_url, std::string userAgent, const std::string& path, long ts, const std::map<std::string, std::string>* headers, long httpCode) const
+{
+  std::string upath{path};
+  if (headers) {
+    auto ent = headers->find("Valid-From");
+    if (ent != headers->end()) {
+      upath += "/" + ent->second;
+    }
+    ent = headers->find("ETag");
+    if (ent != headers->end()) {
+      upath += "/" + ent->second;
+    }
+  }
+  upath.erase(remove(upath.begin(), upath.end(), '\"'), upath.end());
+  return fmt::format("CcdbDownloader finished transfer {}{}{} for {} (agent_id: {}) with http code: {}", host_url, (host_url.back() == '/') ? "" : "/", upath, (ts < 0) ? getCurrentTimestamp() : ts, userAgent, httpCode);
+}
+
 } // namespace o2

CCDB/src/CcdbApi.cxx

@@ -1533,6 +1533,7 @@ void CcdbApi::scheduleDownload(RequestContext& requestContext, size_t* requestCo
   data->path = requestContext.path;
   data->timestamp = requestContext.timestamp;
   data->localContentCallback = localContentCallback;
+  data->userAgent = mUniqueAgentID;
 
   curl_easy_setopt(curl_handle, CURLOPT_URL, fullUrl.c_str());
   initCurlOptionsForRetrieve(curl_handle, (void*)(&data->hoPair), writeCallback, false);
@@ -1570,7 +1571,7 @@ void CcdbApi::releaseNamedSemaphore(boost::interprocess::named_semaphore* sem, s
 }
 
 void CcdbApi::getFromSnapshot(bool createSnapshot, std::string const& path,
-                              long timestamp, std::map<std::string, std::string> headers,
+                              long timestamp, std::map<std::string, std::string>& headers,
                               std::string& snapshotpath, o2::pmr::vector<char>& dest, int& fromSnapshot, std::string const& etag) const
 {
   if (createSnapshot) { // create named semaphore
@@ -1580,9 +1581,10 @@ void CcdbApi::getFromSnapshot(bool createSnapshot, std::string const& path,
       logStream << "CCDB-access[" << getpid() << "] of " << mUniqueAgentID << " to " << path << " timestamp " << timestamp << " for load to memory\n";
     }
   }
-
   if (mInSnapshotMode) { // file must be there, otherwise a fatal will be produced;
-    loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), &headers);
+    if (etag.empty()) {
+      loadFileToMemory(dest, getSnapshotFile(mSnapshotTopPath, path), &headers);
+    }
     fromSnapshot = 1;
   } else if (mPreferSnapshotCache && std::filesystem::exists(snapshotpath)) {
     // if file is available, use it, otherwise cache it below from the server. Do this only when etag is empty since otherwise the object was already fetched and cached
@@ -1671,8 +1673,6 @@ void CcdbApi::vectoredLoadFileToMemory(std::vector<RequestContext>& requestConte
     // navigateSourcesAndLoadFile either retrieves file from snapshot immediately, or schedules it to be downloaded when mDownloader->runLoop is ran at a later time
     auto& requestContext = requestContexts.at(i);
     navigateSourcesAndLoadFile(requestContext, fromSnapshots.at(i), &requestCounter);
-    logReading(requestContext.path, requestContext.timestamp, &requestContext.headers,
-               fmt::format("{}{}", requestContext.considerSnapshot ? "load to memory" : "retrieve", fromSnapshots.at(i) ? " from snapshot" : ""));
   }
 
   // Download the rest
@@ -1683,12 +1683,12 @@ void CcdbApi::vectoredLoadFileToMemory(std::vector<RequestContext>& requestConte
   // Save snapshots
   for (int i = 0; i < requestContexts.size(); i++) {
     auto& requestContext = requestContexts.at(i);
+    logReading(requestContext.path, requestContext.timestamp, &requestContext.headers,
+               fmt::format("{}{}", requestContext.considerSnapshot ? "load to memory" : "retrieve", fromSnapshots.at(i) ? " from snapshot" : ""));
     if (!requestContext.dest.empty()) {
       if (requestContext.considerSnapshot && fromSnapshots.at(i) != 2) {
         saveSnapshot(requestContext);
       }
-    } else {
-      LOG(warning) << "Did not receive content for " << requestContext.path << "\n"; // Temporarily demoted to warning, since it floods the infologger
     }
   }
 }

CCDB/test/testCcdbApiDownloader.cxx

@@ -381,5 +381,15 @@ BOOST_AUTO_TEST_CASE(external_loop_test)
   delete uvLoop;
 }
 
+BOOST_AUTO_TEST_CASE(trim_host_url_test)
+{
+  CCDBDownloader downloader;
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080") == "http://localhost:8080");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost") == "http://localhost");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/some/path") == "http://localhost:8080");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost/some/path") == "http://localhost");
+  BOOST_CHECK(downloader.trimHostUrl("http://localhost:8080/Task/Detector/1?HTTPOnly=true") == "http://localhost:8080");
+}
+
 } // namespace ccdb
 } // namespace o2

Common/Constants/include/CommonConstants/PhysicsConstants.h

@@ -12,40 +12,167 @@
 /// \file PhysicsConstants.h
 /// \brief Header to collect physics constants
 /// \author ruben.shahoyan@cern.ch
+/// \author Vít Kučera <vit.kucera@cern.ch>, Inha University
+/// \note Use the make_pdg_header.py script to generate the enums and mass declarations.
 
 #ifndef ALICEO2_PHYSICSCONSTANTS_H_
 #define ALICEO2_PHYSICSCONSTANTS_H_
 
-namespace o2
-{
-namespace constants
-{
-namespace physics
+namespace o2::constants::physics
 {
 // particles masses
-constexpr float MassPhoton = 0.0;
-constexpr float MassElectron = 0.000511;
-constexpr float MassMuon = 0.105658;
-constexpr float MassPionCharged = 0.139570;
-constexpr float MassPionNeutral = 0.134976;
-constexpr float MassKaonCharged = 0.493677;
-constexpr float MassKaonNeutral = 0.497648;
-constexpr float MassProton = 0.938272;
-constexpr float MassLambda = 1.115683;
-constexpr float MassDeuteron = 1.8756129;
-constexpr float MassTriton = 2.8089211;
-constexpr float MassHelium3 = 2.8083916;
-constexpr float MassAlpha = 3.7273794;
-constexpr float MassHyperTriton = 2.992;
-constexpr float MassHyperhydrog4 = 3.931;
-constexpr float MassHyperhelium4 = 3.9218;
-constexpr float MassXiMinus = 1.32171;
-constexpr float MassOmegaMinus = 1.67245;
+
+// BEGINNING OF THE GENERATED BLOCK.
+// DO NOT EDIT THIS BLOCK DIRECTLY!
+// It has been generated by the make_pdg_header.py script.
+// For modifications, edit the script and generate this block again.
+
+/// \brief Declarations of named PDG codes of particles missing in ROOT PDG_t
+/// \note Follow kCamelCase naming convention
+/// \link https://root.cern/doc/master/TPDGCode_8h.html
+enum Pdg {
+  kB0 = 511,
+  kB0Bar = -511,
+  kBPlus = 521,
+  kBS = 531,
+  kBSBar = -531,
+  kD0 = 421,
+  kD0Bar = -421,
+  kDMinus = -411,
+  kDPlus = 411,
+  kDS = 431,
+  kDSBar = -431,
+  kDStar = 413,
+  kChiC1 = 20443,
+  kJPsi = 443,
+  kLambdaB0 = 5122,
+  kLambdaCPlus = 4122,
+  kOmegaC0 = 4332,
+  kPhi = 333,
+  kSigmaC0 = 4112,
+  kSigmaCPlusPlus = 4222,
+  kX3872 = 9920443,
+  kXi0 = 3322,
+  kXiB0 = 5232,
+  kXiCCPlusPlus = 4422,
+  kXiCPlus = 4232,
+  kXiCZero = 4132,
+  kDeuteron = 1000010020,
+  kTriton = 1000010030,
+  kHelium3 = 1000020030,
+  kAlpha = 1000020040,
+  kHyperTriton = 1010010030,
+  kHyperHydrogen4 = 1010010040,
+  kHyperHelium4 = 1010020040
+};
+
+/// \brief Declarations of masses for additional particles
+constexpr double MassB0 = 5.27953;
+constexpr double MassB0Bar = 5.27953;
+constexpr double MassBPlus = 5.27915;
+constexpr double MassBS = 5.3663;
+constexpr double MassBSBar = 5.3663;
+constexpr double MassD0 = 1.86484;
+constexpr double MassD0Bar = 1.86484;
+constexpr double MassDMinus = 1.86962;
+constexpr double MassDPlus = 1.86962;
+constexpr double MassDS = 1.9685;
+constexpr double MassDSBar = 1.9685;
+constexpr double MassDStar = 2.01027;
+constexpr double MassChiC1 = 3.51066;
+constexpr double MassJPsi = 3.096916;
+constexpr double MassLambdaB0 = 5.6202;
+constexpr double MassLambdaCPlus = 2.28646;
+constexpr double MassOmegaC0 = 2.6975;
+constexpr double MassPhi = 1.019455;
+constexpr double MassSigmaC0 = 2.45376;
+constexpr double MassSigmaCPlusPlus = 2.45402;
+constexpr double MassX3872 = 3.87165;
+constexpr double MassXi0 = 1.31486;
+constexpr double MassXiB0 = 5.7924;
+constexpr double MassXiCCPlusPlus = 3.59798;
+constexpr double MassXiCPlus = 2.4679;
+constexpr double MassXiCZero = 2.471;
+constexpr double MassDeuteron = 1.87561294257;
+constexpr double MassTriton = 2.80892113298;
+constexpr double MassHelium3 = 2.80839160743;
+constexpr double MassAlpha = 3.7273794066;
+constexpr double MassHyperTriton = 2.99131;
+constexpr double MassHyperHydrogen4 = 3.9226;
+constexpr double MassHyperHelium4 = 3.9217;
+
+/// \brief Declarations of masses for particles in ROOT PDG_t
+constexpr double MassDown = 0.0048;
+constexpr double MassDownBar = 0.0048;
+constexpr double MassUp = 0.0024;
+constexpr double MassUpBar = 0.0024;
+constexpr double MassStrange = 0.104;
+constexpr double MassStrangeBar = 0.104;
+constexpr double MassCharm = 1.27;
+constexpr double MassCharmBar = 1.27;
+constexpr double MassBottom = 4.68;
+constexpr double MassBottomBar = 4.68;
+constexpr double MassTop = 171.2;
+constexpr double MassTopBar = 171.2;
+constexpr double MassGluon = 0.0;
+constexpr double MassElectron = 0.00051099891;
+constexpr double MassPositron = 0.00051099891;
+constexpr double MassNuE = 0.0;
+constexpr double MassNuEBar = 0.0;
+constexpr double MassMuonMinus = 0.105658;
+constexpr double MassMuonPlus = 0.105658;
+constexpr double MassNuMu = 0.0;
+constexpr double MassNuMuBar = 0.0;
+constexpr double MassTauMinus = 1.77684;
+constexpr double MassTauPlus = 1.77684;
+constexpr double MassNuTau = 0.0;
+constexpr double MassNuTauBar = 0.0;
+constexpr double MassGamma = 0.0;
+constexpr double MassZ0 = 91.187;
+constexpr double MassWPlus = 80.398;
+constexpr double MassWMinus = 80.398;
+constexpr double MassPi0 = 0.134977;
+constexpr double MassK0Long = 0.497614;
+constexpr double MassPiPlus = 0.13957;
+constexpr double MassPiMinus = 0.13957;
+constexpr double MassProton = 0.938272;
+constexpr double MassProtonBar = 0.938272;
+constexpr double MassNeutron = 0.939565;
+constexpr double MassNeutronBar = 0.939565;
+constexpr double MassK0Short = 0.497614;
+constexpr double MassK0 = 0.497614;
+constexpr double MassK0Bar = 0.497614;
+constexpr double MassKPlus = 0.493677;
+constexpr double MassKMinus = 0.493677;
+constexpr double MassLambda0 = 1.11568;
+constexpr double MassLambda0Bar = 1.11568;
+constexpr double MassLambda1520 = 1.5195;
+constexpr double MassSigmaMinus = 1.19744;
+constexpr double MassSigmaBarPlus = 1.19744;
+constexpr double MassSigmaPlus = 1.18937;
+constexpr double MassSigmaBarMinus = 1.18937;
+constexpr double MassSigma0 = 1.192642;
+constexpr double MassSigma0Bar = 1.192642;
+constexpr double MassXiMinus = 1.32171;
+constexpr double MassXiPlusBar = 1.32171;
+constexpr double MassOmegaMinus = 1.67245;
+constexpr double MassOmegaPlusBar = 1.67245;
+
+// END OF THE GENERATED BLOCK
+
+// legacy names
+constexpr double MassPhoton = MassGamma;
+constexpr double MassMuon = MassMuonMinus;
+constexpr double MassPionCharged = MassPiPlus;
+constexpr double MassPionNeutral = MassPi0;
+constexpr double MassKaonCharged = MassKPlus;
+constexpr double MassKaonNeutral = MassK0;
+constexpr double MassLambda = MassLambda0;
+constexpr double MassHyperhydrog4 = MassHyperHydrogen4;
+constexpr double MassHyperhelium4 = MassHyperHelium4;
 
 constexpr float LightSpeedCm2S = 299792458.e2;           // C in cm/s
 constexpr float LightSpeedCm2NS = LightSpeedCm2S * 1e-9; // C in cm/ns
-} // namespace physics
-} // namespace constants
-} // namespace o2
+} // namespace o2::constants::physics
 
 #endif