VER: Release 0.38.1

CHANGELOG.md

@@ -1,5 +1,22 @@
 # Changelog
 
+
+## 0.38.1 - 2025-06-25
+
+### Enhancements
+- Added `range_by_schema` field to `DatasetRange` struct
+- Changed historical `TimeseriesGetRange` and `TimeseriesGetRangeToFile` methods to use
+  a `POST` request to allow for requesting supported maximum of 2000 symbols
+- Added logging around `Historical::BatchDownload`
+- Changed the following Venue, Publisher, and Dataset descriptions:
+  - "ICE Futures Europe (Financials)" renamed to "ICE Europe Financials"
+  - "ICE Futures Europe (Commodities)" renamed to "ICE Europe Commodities"
+
+### Bug fixes
+- Fixed handling of `null` `last_modified_date` in `MetadataGetDatasetCondition`
+  response
+- Fixed default `ShouldLog` implementation
+
 ## 0.38.0 - 2025-06-10
 
 ### Enhancements
@@ -161,8 +178,8 @@ upgrading data to version 3.
 ## 0.31.0 - 2025-03-18
 
 ### Enhancements
-- Added new venues, datasets, and publishers for ICE Futures US, ICE Futures Europe
-  (Financial products), Eurex, and European Energy Exchange (EEX)
+- Added new venues, datasets, and publishers for ICE Futures US, ICE Europe Financials
+  products, Eurex, and European Energy Exchange (EEX)
 
 ## 0.30.0 - 2025-02-11
 

CMakeLists.txt

@@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24..4.0)
 
 project(
   databento
-  VERSION 0.38.0
+  VERSION 0.38.1
   LANGUAGES CXX
   DESCRIPTION "Official Databento client library"
 )

include/databento/datetime.hpp

@@ -32,4 +32,22 @@ struct DateTimeRange {
   T end;
 };
 using DateRange = DateTimeRange<std::string>;
+
+template <typename T>
+inline bool operator==(const DateTimeRange<T>& lhs,
+                       const DateTimeRange<T>& rhs) {
+  return lhs.start == rhs.start && lhs.end == rhs.end;
+}
+template <typename T>
+inline bool operator!=(const DateTimeRange<T>& lhs,
+                       const DateTimeRange<T>& rhs) {
+  return !(lhs == rhs);
+}
+
+std::string ToString(const DateTimeRange<std::string>& dt_range);
+std::ostream& operator<<(std::ostream& stream,
+                         const DateTimeRange<std::string>& dt_range);
+std::string ToString(const DateTimeRange<UnixNanos>& dt_range);
+std::ostream& operator<<(std::ostream& stream,
+                         const DateTimeRange<UnixNanos>& dt_range);
 }  // namespace databento

include/databento/detail/http_client.hpp

@@ -24,15 +24,23 @@ class HttpClient {
   nlohmann::json GetJson(const std::string& path,
                          const httplib::Params& params);
   nlohmann::json PostJson(const std::string& path,
-                          const httplib::Params& params);
+                          const httplib::Params& form_params);
   void GetRawStream(const std::string& path, const httplib::Params& params,
                     const httplib::ContentReceiver& callback);
+  void PostRawStream(const std::string& path,
+                     const httplib::Params& form_params,
+                     const httplib::ContentReceiver& callback);
 
  private:
+  static bool IsErrorStatus(int status_code);
+  static httplib::ResponseHandler MakeStreamResponseHandler(int& out_status);
+  static void CheckStatusAndStreamRes(const std::string& path, int status_code,
+                                      std::string&& err_body,
+                                      const httplib::Result& res);
+
   nlohmann::json CheckAndParseResponse(const std::string& path,
                                        httplib::Result&& res) const;
   void CheckWarnings(const httplib::Response& response) const;
-  static bool IsErrorStatus(int status_code);
 
   static const httplib::Headers kHeaders;
 

include/databento/detail/json_helpers.hpp

@@ -4,6 +4,7 @@
 #include <nlohmann/json.hpp>
 
 #include <map>  // multimap
+#include <optional>
 #include <string>
 #include <string_view>
 #include <vector>
@@ -79,6 +80,10 @@ template <>
 std::string ParseAt(std::string_view endpoint, const nlohmann::json& json,
                     std::string_view key);
 template <>
+std::optional<std::string> ParseAt(std::string_view endpoint,
+                                   const nlohmann::json& json,
+                                   std::string_view key);
+template <>
 std::uint64_t ParseAt(std::string_view endpoint, const nlohmann::json& json,
                       std::string_view key);
 template <>

include/databento/historical.hpp

@@ -230,8 +230,6 @@ class Historical {
   using HttplibParams = std::multimap<std::string, std::string>;
 
   BatchJob BatchSubmitJob(const HttplibParams& params);
-  void StreamToFile(const std::string& url_path, const HttplibParams& params,
-                    const std::filesystem::path& file_path);
   void DownloadFile(const std::string& url,
                     const std::filesystem::path& output_path);
   std::vector<BatchJob> BatchListJobs(const HttplibParams& params);

include/databento/log.hpp

@@ -37,7 +37,7 @@ class ConsoleLogReceiver : public ILogReceiver {
 
   void Receive(LogLevel level, const std::string& msg) override;
   bool ShouldLog(databento::LogLevel level) const override {
-    return level > min_level_;
+    return level >= min_level_;
   }
 
  private:

include/databento/metadata.hpp

@@ -2,9 +2,12 @@
 
 #include <cstdint>
 #include <map>
+#include <optional>
 #include <ostream>
 #include <string>
+#include <unordered_map>
 
+#include "databento/datetime.hpp"
 #include "databento/enums.hpp"  // FeedMode, DatasetCondition, Schema
 
 namespace databento {
@@ -28,12 +31,13 @@ struct UnitPricesForMode {
 struct DatasetConditionDetail {
   std::string date;
   DatasetCondition condition;
-  std::string last_modified_date;
+  std::optional<std::string> last_modified_date;
 };
 
 struct DatasetRange {
   std::string start;
   std::string end;
+  std::map<Schema, DateTimeRange<std::string>> range_by_schema;
 };
 
 inline bool operator==(const PublisherDetail& lhs, const PublisherDetail& rhs) {
@@ -71,7 +75,8 @@ inline bool operator!=(const DatasetConditionDetail& lhs,
 }
 
 inline bool operator==(const DatasetRange& lhs, const DatasetRange& rhs) {
-  return lhs.start == rhs.start && lhs.end == rhs.end;
+  return lhs.start == rhs.start && lhs.end == rhs.end &&
+         lhs.range_by_schema == rhs.range_by_schema;
 }
 inline bool operator!=(const DatasetRange& lhs, const DatasetRange& rhs) {
   return !(lhs == rhs);

include/databento/publishers.hpp

@@ -81,7 +81,7 @@ enum class Venue : std::uint16_t {
   Bato = 36,
   // MEMX Options
   Mxop = 37,
-  // ICE Futures Europe (Commodities)
+  // ICE Europe Commodities
   Ifeu = 38,
   // ICE Endex
   Ndex = 39,
@@ -103,7 +103,7 @@ enum class Venue : std::uint16_t {
   Equs = 47,
   // ICE Futures US
   Ifus = 48,
-  // ICE Futures Europe (Financials)
+  // ICE Europe Financials
   Ifll = 49,
   // Eurex Exchange
   Xeur = 50,
@@ -167,7 +167,7 @@ enum class Dataset : std::uint16_t {
   XnasQbbo = 26,
   // Nasdaq NLS
   XnasNls = 27,
-  // ICE Futures Europe (Commodities) iMpact
+  // ICE Europe Commodities iMpact
   IfeuImpact = 28,
   // ICE Endex iMpact
   NdexImpact = 29,
@@ -185,7 +185,7 @@ enum class Dataset : std::uint16_t {
   EqusMini = 35,
   // ICE Futures US iMpact
   IfusImpact = 36,
-  // ICE Futures Europe (Financials) iMpact
+  // ICE Europe Financials iMpact
   IfllImpact = 37,
   // Eurex EOBI
   XeurEobi = 38,
@@ -307,7 +307,7 @@ enum class Publisher : std::uint16_t {
   EqusPlusFiny = 55,
   // Databento US Equities Plus - FINRA/Nasdaq TRF Chicago
   EqusPlusFinc = 56,
-  // ICE Futures Europe (Commodities)
+  // ICE Europe Commodities
   IfeuImpactIfeu = 57,
   // ICE Endex
   NdexImpactNdex = 58,
@@ -361,7 +361,7 @@ enum class Publisher : std::uint16_t {
   XnasBasicFinn = 82,
   // Nasdaq Basic - FINRA/Nasdaq TRF Chicago
   XnasBasicFinc = 83,
-  // ICE Futures Europe - Off-Market Trades
+  // ICE Europe - Off-Market Trades
   IfeuImpactXoff = 84,
   // ICE Endex - Off-Market Trades
   NdexImpactXoff = 85,
@@ -391,9 +391,9 @@ enum class Publisher : std::uint16_t {
   IfusImpactIfus = 97,
   // ICE Futures US - Off-Market Trades
   IfusImpactXoff = 98,
-  // ICE Futures Europe (Financials)
+  // ICE Europe Financials
   IfllImpactIfll = 99,
-  // ICE Futures Europe (Financials) - Off-Market Trades
+  // ICE Europe Financials - Off-Market Trades
   IfllImpactXoff = 100,
   // Eurex EOBI
   XeurEobiXeur = 101,

pkg/PKGBUILD

@@ -1,7 +1,7 @@
 # Maintainer: Databento <support@databento.com>
 _pkgname=databento-cpp
 pkgname=databento-cpp-git
-pkgver=0.38.0
+pkgver=0.38.1
 pkgrel=1
 pkgdesc="Official C++ client for Databento"
 arch=('any')

src/datetime.cpp

@@ -10,6 +10,7 @@
 #include <sstream>  // ostringstream
 
 #include "databento/constants.hpp"  // kUndefTimestamp
+#include "stream_op_helper.hpp"
 
 namespace databento {
 std::string ToIso8601(UnixNanos unix_nanos) {
@@ -61,4 +62,32 @@ std::string DateFromIso8601Int(std::uint32_t date_int) {
          << std::setfill('0') << std::setw(2) << day;
   return out_ss.str();
 }
+
+std::string ToString(const DateTimeRange<std::string>& dt_range) {
+  return MakeString(dt_range);
+}
+std::ostream& operator<<(std::ostream& stream,
+                         const DateTimeRange<std::string>& dt_range) {
+  return StreamOpBuilder{stream}
+      .SetSpacer(" ")
+      .SetTypeName("DateTimeRange")
+      .Build()
+      .AddField("start", dt_range.start)
+      .AddField("end", dt_range.end)
+      .Finish();
+}
+
+std::string ToString(const DateTimeRange<UnixNanos>& dt_range) {
+  return MakeString(dt_range);
+}
+std::ostream& operator<<(std::ostream& stream,
+                         const DateTimeRange<UnixNanos>& dt_range) {
+  return StreamOpBuilder{stream}
+      .SetSpacer(" ")
+      .SetTypeName("DateTimeRange")
+      .Build()
+      .AddField("start", dt_range.start)
+      .AddField("end", dt_range.end)
+      .Finish();
+}
 }  // namespace databento

src/detail/http_client.cpp

@@ -53,25 +53,64 @@ void HttpClient::GetRawStream(const std::string& path,
   const std::string full_path = httplib::append_query_params(path, params);
   std::string err_body{};
   int err_status{};
-  const httplib::Result res = client_.Get(
-      full_path,
-      [&err_status](const httplib::Response& resp) {
-        if (HttpClient::IsErrorStatus(resp.status)) {
-          err_status = resp.status;
-        }
-        return true;
-      },
-      [&callback, &err_body, &err_status](const char* data,
-                                          std::size_t length) {
-        // if an error response was received, read all content into err_status
-        if (err_status > 0) {
-          err_body.append(data, length);
-          return true;
-        }
-        return callback(data, length);
-      });
-  if (err_status > 0) {
-    throw HttpResponseError{path, err_status, std::move(err_body)};
+  const httplib::Result res =
+      client_.Get(full_path, MakeStreamResponseHandler(err_status),
+                  [&callback, &err_body, &err_status](const char* data,
+                                                      std::size_t length) {
+                    // if an error response was received, read all content into
+                    // err_body
+                    if (err_status > 0) {
+                      err_body.append(data, length);
+                      return true;
+                    }
+                    return callback(data, length);
+                  });
+  CheckStatusAndStreamRes(path, err_status, std::move(err_body), res);
+}
+
+void HttpClient::PostRawStream(const std::string& path,
+                               const httplib::Params& form_params,
+                               const httplib::ContentReceiver& callback) {
+  std::string err_body{};
+  int err_status{};
+  httplib::Request req;
+  req.method = "POST";
+  req.set_header("Content-Type", "application/x-www-form-urlencoded");
+  req.path = path;
+  req.body = httplib::detail::params_to_query_str(form_params);
+  req.response_handler = MakeStreamResponseHandler(err_status);
+  req.content_receiver = [&callback, &err_body, &err_status](
+                             const char* data, std::size_t length,
+                             std::uint64_t, std::uint64_t) {
+    // if an error response was received, read all content into
+    // err_body
+    if (err_status > 0) {
+      err_body.append(data, length);
+      return true;
+    }
+    return callback(data, length);
+  };
+  // NOLINTNEXTLINE(clang-analyzer-unix.BlockInCriticalSection): dependency code
+  const httplib::Result res = client_.send(req);
+  CheckStatusAndStreamRes(path, err_status, std::move(err_body), res);
+}
+
+httplib::ResponseHandler HttpClient::MakeStreamResponseHandler(
+    int& out_status) {
+  return [&out_status](const httplib::Response& resp) {
+    if (HttpClient::IsErrorStatus(resp.status)) {
+      out_status = resp.status;
+    }
+    return true;
+  };
+}
+
+void HttpClient::CheckStatusAndStreamRes(const std::string& path,
+                                         int status_code,
+                                         std::string&& err_body,
+                                         const httplib::Result& res) {
+  if (status_code > 0) {
+    throw HttpResponseError{path, status_code, std::move(err_body)};
   }
   if (res.error() != httplib::Error::Success &&
       // canceled happens if `callback` returns false, which is based on the

src/detail/json_helpers.cpp

@@ -1,6 +1,7 @@
 #include "databento/detail/json_helpers.hpp"
 
 #include <numeric>  // accumulate
+#include <optional>
 #include <sstream>  // istringstream
 #include <string>
 
@@ -49,6 +50,17 @@ bool ParseAt(std::string_view endpoint, const nlohmann::json& json,
 template <>
 std::string ParseAt(std::string_view endpoint, const nlohmann::json& json,
                     std::string_view key) {
+  const auto s = ParseAt<std::optional<std::string>>(endpoint, json, key);
+  if (s) {
+    return *s;
+  }
+  return {};
+}
+
+template <>
+std::optional<std::string> ParseAt(std::string_view endpoint,
+                                   const nlohmann::json& json,
+                                   std::string_view key) {
   const auto& val_json = CheckedAt(endpoint, json, key);
   if (val_json.is_null()) {
     return {};