From c41c5dd1535b4ea01be5cfb123f589b3c46a8e53 Mon Sep 17 00:00:00 2001 From: Roberto Mesado Date: Tue, 10 Jun 2025 18:34:04 -0700 Subject: [PATCH 1/7] MOD: ICE renaming --- CHANGELOG.md | 4 ++-- include/databento/publishers.hpp | 24 ++++++++++++------------ 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ea9fe88..67afa00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -161,8 +161,8 @@ upgrading data to version 3. ## 0.31.0 - 2025-03-18 ### Enhancements -- Added new venues, datasets, and publishers for ICE Futures US, ICE Futures Europe - (Financial products), Eurex, and European Energy Exchange (EEX) +- Added new venues, datasets, and publishers for ICE US, ICE Europe Financials + products, Eurex, and European Energy Exchange (EEX) ## 0.30.0 - 2025-02-11 diff --git a/include/databento/publishers.hpp b/include/databento/publishers.hpp index 9a20062..97adae0 100644 --- a/include/databento/publishers.hpp +++ b/include/databento/publishers.hpp @@ -81,7 +81,7 @@ enum class Venue : std::uint16_t { Bato = 36, // MEMX Options Mxop = 37, - // ICE Futures Europe (Commodities) + // ICE Europe Commodities Ifeu = 38, // ICE Endex Ndex = 39, @@ -101,9 +101,9 @@ enum class Venue : std::uint16_t { Aspi = 46, // Databento US Equities - Consolidated Equs = 47, - // ICE Futures US + // ICE US Ifus = 48, - // ICE Futures Europe (Financials) + // ICE Europe Financials Ifll = 49, // Eurex Exchange Xeur = 50, @@ -167,7 +167,7 @@ enum class Dataset : std::uint16_t { XnasQbbo = 26, // Nasdaq NLS XnasNls = 27, - // ICE Futures Europe (Commodities) iMpact + // ICE Europe Commodities iMpact IfeuImpact = 28, // ICE Endex iMpact NdexImpact = 29, @@ -183,9 +183,9 @@ enum class Dataset : std::uint16_t { XnysTradesbbo = 34, // Databento US Equities Mini EqusMini = 35, - // ICE Futures US iMpact + // ICE US iMpact IfusImpact = 36, - // ICE Futures Europe (Financials) iMpact + // ICE Europe Financials iMpact IfllImpact = 37, // Eurex EOBI XeurEobi = 38, @@ -307,7 +307,7 @@ enum class Publisher : std::uint16_t { EqusPlusFiny = 55, // Databento US Equities Plus - FINRA/Nasdaq TRF Chicago EqusPlusFinc = 56, - // ICE Futures Europe (Commodities) + // ICE Europe Commodities IfeuImpactIfeu = 57, // ICE Endex NdexImpactNdex = 58, @@ -361,7 +361,7 @@ enum class Publisher : std::uint16_t { XnasBasicFinn = 82, // Nasdaq Basic - FINRA/Nasdaq TRF Chicago XnasBasicFinc = 83, - // ICE Futures Europe - Off-Market Trades + // ICE Europe - Off-Market Trades IfeuImpactXoff = 84, // ICE Endex - Off-Market Trades NdexImpactXoff = 85, @@ -387,13 +387,13 @@ enum class Publisher : std::uint16_t { EqusMiniEqus = 95, // NYSE Trades - Consolidated XnysTradesEqus = 96, - // ICE Futures US + // ICE US IfusImpactIfus = 97, - // ICE Futures US - Off-Market Trades + // ICE US - Off-Market Trades IfusImpactXoff = 98, - // ICE Futures Europe (Financials) + // ICE Europe Financials IfllImpactIfll = 99, - // ICE Futures Europe (Financials) - Off-Market Trades + // ICE Europe Financials - Off-Market Trades IfllImpactXoff = 100, // Eurex EOBI XeurEobiXeur = 101, From fd0a27d746c8eef3716a253fbe827120cfdff175 Mon Sep 17 00:00:00 2001 From: Roberto Mesado Date: Mon, 16 Jun 2025 13:39:01 -0700 Subject: [PATCH 2/7] MOD: ICE US to ICE Futures US --- CHANGELOG.md | 2 +- include/databento/publishers.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 67afa00..100b07e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -161,7 +161,7 @@ upgrading data to version 3. ## 0.31.0 - 2025-03-18 ### Enhancements -- Added new venues, datasets, and publishers for ICE US, ICE Europe Financials +- Added new venues, datasets, and publishers for ICE Futures US, ICE Europe Financials products, Eurex, and European Energy Exchange (EEX) ## 0.30.0 - 2025-02-11 diff --git a/include/databento/publishers.hpp b/include/databento/publishers.hpp index 97adae0..4e7cede 100644 --- a/include/databento/publishers.hpp +++ b/include/databento/publishers.hpp @@ -101,7 +101,7 @@ enum class Venue : std::uint16_t { Aspi = 46, // Databento US Equities - Consolidated Equs = 47, - // ICE US + // ICE Futures US Ifus = 48, // ICE Europe Financials Ifll = 49, @@ -183,7 +183,7 @@ enum class Dataset : std::uint16_t { XnysTradesbbo = 34, // Databento US Equities Mini EqusMini = 35, - // ICE US iMpact + // ICE Futures US iMpact IfusImpact = 36, // ICE Europe Financials iMpact IfllImpact = 37, @@ -387,9 +387,9 @@ enum class Publisher : std::uint16_t { EqusMiniEqus = 95, // NYSE Trades - Consolidated XnysTradesEqus = 96, - // ICE US + // ICE Futures US IfusImpactIfus = 97, - // ICE US - Off-Market Trades + // ICE Futures US - Off-Market Trades IfusImpactXoff = 98, // ICE Europe Financials IfllImpactIfll = 99, From 48b7630ac076dc49f89b424e39ec1ba0861d2350 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 17 Jun 2025 14:53:51 +0100 Subject: [PATCH 3/7] ADD: Add per schema dataset range in C++ and Rust --- CHANGELOG.md | 9 +++++++++ include/databento/datetime.hpp | 18 ++++++++++++++++++ include/databento/metadata.hpp | 6 +++++- src/datetime.cpp | 29 +++++++++++++++++++++++++++++ src/historical.cpp | 25 ++++++++++++++++++++++++- src/metadata.cpp | 13 ++++++++++++- src/stream_op_helper.hpp | 13 +++++++++++++ tests/src/historical_tests.cpp | 18 ++++++++++++++++-- tests/src/metadata_tests.cpp | 21 ++++++++++++++++----- 9 files changed, 142 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 100b07e..6133f84 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,14 @@ # Changelog + +## 0.38.1 - TBD + +### Enhancements +- Added `range_by_schema` field to `DatasetRange` struct +- Changed the following Venue, Publisher, and Dataset descriptions: + - "ICE Futures Europe (Financials)" renamed to "ICE Europe Financials" + - "ICE Futures Europe (Commodities)" renamed to "ICE Europe Commodities" + ## 0.38.0 - 2025-06-10 ### Enhancements diff --git a/include/databento/datetime.hpp b/include/databento/datetime.hpp index 60f1522..9ff8a0c 100644 --- a/include/databento/datetime.hpp +++ b/include/databento/datetime.hpp @@ -32,4 +32,22 @@ struct DateTimeRange { T end; }; using DateRange = DateTimeRange; + +template +inline bool operator==(const DateTimeRange& lhs, + const DateTimeRange& rhs) { + return lhs.start == rhs.start && lhs.end == rhs.end; +} +template +inline bool operator!=(const DateTimeRange& lhs, + const DateTimeRange& rhs) { + return !(lhs == rhs); +} + +std::string ToString(const DateTimeRange& dt_range); +std::ostream& operator<<(std::ostream& stream, + const DateTimeRange& dt_range); +std::string ToString(const DateTimeRange& dt_range); +std::ostream& operator<<(std::ostream& stream, + const DateTimeRange& dt_range); } // namespace databento diff --git a/include/databento/metadata.hpp b/include/databento/metadata.hpp index c79f321..988e3aa 100644 --- a/include/databento/metadata.hpp +++ b/include/databento/metadata.hpp @@ -4,7 +4,9 @@ #include #include #include +#include +#include "databento/datetime.hpp" #include "databento/enums.hpp" // FeedMode, DatasetCondition, Schema namespace databento { @@ -34,6 +36,7 @@ struct DatasetConditionDetail { struct DatasetRange { std::string start; std::string end; + std::unordered_map> range_by_schema; }; inline bool operator==(const PublisherDetail& lhs, const PublisherDetail& rhs) { @@ -71,7 +74,8 @@ inline bool operator!=(const DatasetConditionDetail& lhs, } inline bool operator==(const DatasetRange& lhs, const DatasetRange& rhs) { - return lhs.start == rhs.start && lhs.end == rhs.end; + return lhs.start == rhs.start && lhs.end == rhs.end && + lhs.range_by_schema == rhs.range_by_schema; } inline bool operator!=(const DatasetRange& lhs, const DatasetRange& rhs) { return !(lhs == rhs); diff --git a/src/datetime.cpp b/src/datetime.cpp index 93e3f2f..4319db2 100644 --- a/src/datetime.cpp +++ b/src/datetime.cpp @@ -10,6 +10,7 @@ #include // ostringstream #include "databento/constants.hpp" // kUndefTimestamp +#include "stream_op_helper.hpp" namespace databento { std::string ToIso8601(UnixNanos unix_nanos) { @@ -61,4 +62,32 @@ std::string DateFromIso8601Int(std::uint32_t date_int) { << std::setfill('0') << std::setw(2) << day; return out_ss.str(); } + +std::string ToString(const DateTimeRange& dt_range) { + return MakeString(dt_range); +} +std::ostream& operator<<(std::ostream& stream, + const DateTimeRange& dt_range) { + return StreamOpBuilder{stream} + .SetSpacer(" ") + .SetTypeName("DateTimeRange") + .Build() + .AddField("start", dt_range.start) + .AddField("end", dt_range.end) + .Finish(); +} + +std::string ToString(const DateTimeRange& dt_range) { + return MakeString(dt_range); +} +std::ostream& operator<<(std::ostream& stream, + const DateTimeRange& dt_range) { + return StreamOpBuilder{stream} + .SetSpacer(" ") + .SetTypeName("DateTimeRange") + .Build() + .AddField("start", dt_range.start) + .AddField("end", dt_range.end) + .Finish(); +} } // namespace databento diff --git a/src/historical.cpp b/src/historical.cpp index 3088183..7053b16 100644 --- a/src/historical.cpp +++ b/src/historical.cpp @@ -1,5 +1,9 @@ #include "databento/historical.hpp" +#include + +#include "databento/publishers.hpp" + #ifndef CPPHTTPLIB_OPENSSL_SUPPORT #define CPPHTTPLIB_OPENSSL_SUPPORT #endif @@ -529,8 +533,27 @@ databento::DatasetRange Historical::MetadataGetDatasetRange( if (!json.is_object()) { throw JsonResponseError::TypeMismatch(kEndpoint, "object", json); } + const auto& schema_json = detail::CheckedAt(kEndpoint, json, "schema"); + if (!schema_json.is_object()) { + throw JsonResponseError::TypeMismatch(kEndpoint, "schema object", json); + } + std::unordered_map> range_by_schema; + for (const auto& schema_item : schema_json.items()) { + if (!schema_item.value().is_object()) { + throw JsonResponseError::TypeMismatch(kEndpoint, "nested schema object", + json); + } + auto start = + detail::ParseAt(kEndpoint, schema_item.value(), "start"); + auto end = + detail::ParseAt(kEndpoint, schema_item.value(), "end"); + range_by_schema.emplace( + FromString(schema_item.key()), + DateTimeRange{std::move(start), std::move(end)}); + } return DatasetRange{detail::ParseAt(kEndpoint, json, "start"), - detail::ParseAt(kEndpoint, json, "end")}; + detail::ParseAt(kEndpoint, json, "end"), + std::move(range_by_schema)}; } static const std::string kMetadataGetRecordCountEndpoint = diff --git a/src/metadata.cpp b/src/metadata.cpp index 6c8b3b6..32513b5 100644 --- a/src/metadata.cpp +++ b/src/metadata.cpp @@ -1,6 +1,7 @@ #include "databento/metadata.hpp" #include +#include #include "stream_op_helper.hpp" @@ -55,12 +56,22 @@ std::string ToString(const DatasetRange& dataset_range) { } std::ostream& operator<<(std::ostream& stream, const DatasetRange& dataset_range) { + std::ostringstream range_by_schema_ss; + auto range_by_schema_helper = StreamOpBuilder{range_by_schema_ss} + .SetSpacer("\n ") + .SetIndent(" ") + .Build(); + for (const auto& [schema, range] : dataset_range.range_by_schema) { + range_by_schema_helper.AddKeyVal(schema, range); + } + range_by_schema_helper.Finish(); return StreamOpBuilder{stream} - .SetSpacer(" ") + .SetSpacer("\n ") .SetTypeName("DatasetRange") .Build() .AddField("start", dataset_range.start) .AddField("end", dataset_range.end) + .AddField("range_by_schema", range_by_schema_ss) .Finish(); } } // namespace databento diff --git a/src/stream_op_helper.hpp b/src/stream_op_helper.hpp index a84f942..951010b 100644 --- a/src/stream_op_helper.hpp +++ b/src/stream_op_helper.hpp @@ -109,6 +109,19 @@ class StreamOpHelper { return *this; } + template + StreamOpHelper& AddKeyVal(const K& key, const V& val) { + if (!is_first_) { + stream_ << ','; + } + stream_ << spacer_ << indent_; + FmtToStream(key); + stream_ << ": "; + FmtToStream(val); + is_first_ = false; + return *this; + } + std::ostream& Finish() { if (spacer_.find('\n') == std::string::npos) { // no spacing required if empty diff --git a/tests/src/historical_tests.cpp b/tests/src/historical_tests.cpp index 858ae3a..83158b7 100644 --- a/tests/src/historical_tests.cpp +++ b/tests/src/historical_tests.cpp @@ -464,8 +464,22 @@ TEST_F(HistoricalTests, TestMetadataListUnitPrices) { } TEST_F(HistoricalTests, TestMetadataGetDatasetRange) { - const nlohmann::json kResp = {{"start", "2017-05-21T00:00:00.000000000Z"}, - {"end", "2022-12-01T00:00:00.000000000Z"}}; + const nlohmann::json kResp = { + {"start", "2017-05-21T00:00:00.000000000Z"}, + {"end", "2022-12-01T00:00:00.000000000Z"}, + {"schema", + { + {"bbo-1m", + {{"start", "2020-08-02T00:00:00.000000000Z"}, + {"end", "2023-03-23T00:00:00.000000000Z"}}}, + {"ohlcv-1s", + {{"start", "2020-08-02T00:00:00.000000000Z"}, + {"end", "2023-03-23T00:00:00.000000000Z"}}}, + {"ohlcv-1m", + {{"start", "2020-08-02T00:00:00.000000000Z"}, + {"end", "2023-03-23T00:00:00.000000000Z"}}}, + + }}}; mock_server_.MockGetJson("/v0/metadata.get_dataset_range", {{"dataset", dataset::kXnasItch}}, kResp); const auto port = mock_server_.ListenOnThread(); diff --git a/tests/src/metadata_tests.cpp b/tests/src/metadata_tests.cpp index 564bad3..78592bb 100644 --- a/tests/src/metadata_tests.cpp +++ b/tests/src/metadata_tests.cpp @@ -13,10 +13,21 @@ TEST(MetadataTests, TestDatasetConditionDetailToString) { } TEST(MetadataTests, TestDatasetRangeToString) { - const DatasetRange target{"2022-05-17T00:00:00.000000000Z", - "2023-01-07T00:00:00.000000000Z"}; - ASSERT_EQ( - ToString(target), - R"(DatasetRange { start = "2022-05-17T00:00:00.000000000Z", end = "2023-01-07T00:00:00.000000000Z" })"); + const DatasetRange target{ + "2022-05-17T00:00:00.000000000Z", + "2023-01-07T00:00:00.000000000Z", + {{Schema::Bbo1M, + {"2020-08-02T00:00:00.000000000Z", "2023-03-23T00:00:00.000000000Z"}}, + {Schema::Bbo1S, + {"2020-08-02T00:00:00.000000000Z", "2023-03-23T00:00:00.000000000Z"}}}}; + ASSERT_EQ(ToString(target), + R"(DatasetRange { + start = "2022-05-17T00:00:00.000000000Z", + end = "2023-01-07T00:00:00.000000000Z", + range_by_schema = { + bbo-1s: DateTimeRange { start = "2020-08-02T00:00:00.000000000Z", end = "2023-03-23T00:00:00.000000000Z" }, + bbo-1m: DateTimeRange { start = "2020-08-02T00:00:00.000000000Z", end = "2023-03-23T00:00:00.000000000Z" } + } +})"); } } // namespace databento::tests From 182a4d1ef69ab77849adc119342d07a2c4f9a8d0 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Thu, 19 Jun 2025 14:13:58 +0100 Subject: [PATCH 4/7] ADD: Add batch download logging --- CHANGELOG.md | 4 ++++ include/databento/log.hpp | 2 +- src/historical.cpp | 22 +++++++++++++++------- tests/src/log_tests.cpp | 7 +++++-- 4 files changed, 25 insertions(+), 10 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6133f84..d139429 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,14 @@ ### Enhancements - Added `range_by_schema` field to `DatasetRange` struct +- Added logging around `Historical::BatchDownload` - Changed the following Venue, Publisher, and Dataset descriptions: - "ICE Futures Europe (Financials)" renamed to "ICE Europe Financials" - "ICE Futures Europe (Commodities)" renamed to "ICE Europe Commodities" +### Bug fixes +- Fixed default `ShouldLog` implementation + ## 0.38.0 - 2025-06-10 ### Enhancements diff --git a/include/databento/log.hpp b/include/databento/log.hpp index d8acf18..1398c3a 100644 --- a/include/databento/log.hpp +++ b/include/databento/log.hpp @@ -37,7 +37,7 @@ class ConsoleLogReceiver : public ILogReceiver { void Receive(LogLevel level, const std::string& msg) override; bool ShouldLog(databento::LogLevel level) const override { - return level > min_level_; + return level >= min_level_; } private: diff --git a/src/historical.cpp b/src/historical.cpp index 7053b16..6d6cb9e 100644 --- a/src/historical.cpp +++ b/src/historical.cpp @@ -1,9 +1,5 @@ #include "databento/historical.hpp" -#include - -#include "databento/publishers.hpp" - #ifndef CPPHTTPLIB_OPENSSL_SUPPORT #define CPPHTTPLIB_OPENSSL_SUPPORT #endif @@ -18,6 +14,7 @@ #include #include #include +#include #include // move #include "databento/constants.hpp" @@ -332,25 +329,36 @@ void Historical::StreamToFile(const std::string& url_path, void Historical::DownloadFile(const std::string& url, const std::filesystem::path& output_path) { - static const std::string kEndpoint = "Historical::DownloadFile"; + static const std::string kMethod = "Historical::DownloadFile"; // extract path from URL const auto protocol_divider = url.find("://"); std::string path; + if (protocol_divider == std::string::npos) { const auto slash = url.find_first_of('/'); if (slash == std::string::npos) { - throw InvalidArgumentError{kEndpoint, "url", "No slashes"}; + throw InvalidArgumentError{kMethod, "url", "No slashes"}; } path = url.substr(slash); } else { const auto slash = url.find('/', protocol_divider + 3); if (slash == std::string::npos) { - throw InvalidArgumentError{kEndpoint, "url", "No slashes"}; + throw InvalidArgumentError{kMethod, "url", "No slashes"}; } path = url.substr(slash); } + std::ostringstream ss; + ss << '[' << kMethod << "] Downloading batch file " << path << " to " + << output_path; + log_receiver_->Receive(LogLevel::Info, ss.str()); StreamToFile(path, {}, output_path); + + if (log_receiver_->ShouldLog(LogLevel::Debug)) { + ss.str(""); + ss << '[' << kMethod << ']' << " Completed download of " << path; + log_receiver_->Receive(LogLevel::Debug, ss.str()); + } } std::vector Historical::MetadataListPublishers() { diff --git a/tests/src/log_tests.cpp b/tests/src/log_tests.cpp index 13d45b0..d14b965 100644 --- a/tests/src/log_tests.cpp +++ b/tests/src/log_tests.cpp @@ -1,9 +1,10 @@ -#include +#include #include #include #include "databento/log.hpp" +#include "gmock/gmock.h" namespace databento::tests { class ConsoleLogReceiverTests : public testing::Test { @@ -24,7 +25,9 @@ TEST_F(ConsoleLogReceiverTests, TestFilter) { const std::string msg = "Something happened"; target_.Receive(LogLevel::Debug, msg); const std::string output = stream_.str(); - ASSERT_TRUE(output.empty()); + EXPECT_TRUE(output.empty()); + target_.Receive(LogLevel::Info, msg); + EXPECT_THAT(stream_.str(), testing::HasSubstr(msg)); } TEST(ILogReceiverTests, TestDefault) { From 158d28cb806a5434a358b993f21d51903046a596 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 24 Jun 2025 09:02:12 -0500 Subject: [PATCH 5/7] FIX: Fix handling of null `last_modified_date` --- CHANGELOG.md | 2 ++ include/databento/detail/json_helpers.hpp | 5 +++++ include/databento/metadata.hpp | 3 ++- src/detail/json_helpers.cpp | 12 ++++++++++++ src/historical.cpp | 10 ++++------ src/stream_op_helper.hpp | 2 +- tests/src/historical_tests.cpp | 5 +++-- tests/src/metadata_tests.cpp | 13 +++++++++---- 8 files changed, 38 insertions(+), 14 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d139429..f9a30a8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,8 @@ - "ICE Futures Europe (Commodities)" renamed to "ICE Europe Commodities" ### Bug fixes +- Fixed handling of `null` `last_modified_date` in `MetadataGetDatasetCondition` + response - Fixed default `ShouldLog` implementation ## 0.38.0 - 2025-06-10 diff --git a/include/databento/detail/json_helpers.hpp b/include/databento/detail/json_helpers.hpp index e54dad2..1ac05f6 100644 --- a/include/databento/detail/json_helpers.hpp +++ b/include/databento/detail/json_helpers.hpp @@ -4,6 +4,7 @@ #include #include // multimap +#include #include #include #include @@ -79,6 +80,10 @@ template <> std::string ParseAt(std::string_view endpoint, const nlohmann::json& json, std::string_view key); template <> +std::optional ParseAt(std::string_view endpoint, + const nlohmann::json& json, + std::string_view key); +template <> std::uint64_t ParseAt(std::string_view endpoint, const nlohmann::json& json, std::string_view key); template <> diff --git a/include/databento/metadata.hpp b/include/databento/metadata.hpp index 988e3aa..54575a0 100644 --- a/include/databento/metadata.hpp +++ b/include/databento/metadata.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -30,7 +31,7 @@ struct UnitPricesForMode { struct DatasetConditionDetail { std::string date; DatasetCondition condition; - std::string last_modified_date; + std::optional last_modified_date; }; struct DatasetRange { diff --git a/src/detail/json_helpers.cpp b/src/detail/json_helpers.cpp index 7abae76..d8d1ea4 100644 --- a/src/detail/json_helpers.cpp +++ b/src/detail/json_helpers.cpp @@ -1,6 +1,7 @@ #include "databento/detail/json_helpers.hpp" #include // accumulate +#include #include // istringstream #include @@ -49,6 +50,17 @@ bool ParseAt(std::string_view endpoint, const nlohmann::json& json, template <> std::string ParseAt(std::string_view endpoint, const nlohmann::json& json, std::string_view key) { + const auto s = ParseAt>(endpoint, json, key); + if (s) { + return *s; + } + return {}; +} + +template <> +std::optional ParseAt(std::string_view endpoint, + const nlohmann::json& json, + std::string_view key) { const auto& val_json = CheckedAt(endpoint, json, key); if (val_json.is_null()) { return {}; diff --git a/src/historical.cpp b/src/historical.cpp index 6d6cb9e..4eeaf65 100644 --- a/src/historical.cpp +++ b/src/historical.cpp @@ -520,12 +520,10 @@ Historical::MetadataGetDatasetCondition(const httplib::Params& params) { if (!detail_json.is_object()) { throw JsonResponseError::TypeMismatch(kEndpoint, "object", detail_json); } - std::string date = - detail::ParseAt(kEndpoint, detail_json, "date"); - const DatasetCondition condition = - detail::FromCheckedAtString(kEndpoint, detail_json, - "condition"); - std::string last_modified_date = detail::ParseAt( + auto date = detail::ParseAt(kEndpoint, detail_json, "date"); + const auto condition = detail::FromCheckedAtString( + kEndpoint, detail_json, "condition"); + auto last_modified_date = detail::ParseAt>( kEndpoint, detail_json, "last_modified_date"); details.emplace_back(DatasetConditionDetail{std::move(date), condition, std::move(last_modified_date)}); diff --git a/src/stream_op_helper.hpp b/src/stream_op_helper.hpp index 951010b..2e5b650 100644 --- a/src/stream_op_helper.hpp +++ b/src/stream_op_helper.hpp @@ -60,7 +60,7 @@ class StreamOpHelper { template void FmtToStream(const std::optional& val) { if (val.has_value()) { - stream_ << *val; + FmtToStream(*val); } else { stream_ << "nullopt"; } diff --git a/tests/src/historical_tests.cpp b/tests/src/historical_tests.cpp index 83158b7..3e201b5 100644 --- a/tests/src/historical_tests.cpp +++ b/tests/src/historical_tests.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include // logic_error #include // move @@ -424,7 +425,7 @@ TEST_F(HistoricalTests, TestMetadataGetDatasetCondition) { {"last_modified_date", "2023-03-01"}}, {{"date", "2022-11-10"}, {"condition", "missing"}, - {"last_modified_date", "2023-03-01"}}}; + {"last_modified_date", nullptr}}}; mock_server_.MockGetJson("/v0/metadata.get_dataset_condition", {{"dataset", dataset::kXnasItch}, {"start_date", "2022-11-06"}, @@ -440,7 +441,7 @@ TEST_F(HistoricalTests, TestMetadataGetDatasetCondition) { {"2022-11-07", DatasetCondition::Available, "2023-03-01"}, {"2022-11-08", DatasetCondition::Degraded, "2023-03-01"}, {"2022-11-09", DatasetCondition::Pending, "2023-03-01"}, - {"2022-11-10", DatasetCondition::Missing, "2023-03-01"}, + {"2022-11-10", DatasetCondition::Missing, std::nullopt}, }; EXPECT_EQ(res, kExp); } diff --git a/tests/src/metadata_tests.cpp b/tests/src/metadata_tests.cpp index 78592bb..a81d10d 100644 --- a/tests/src/metadata_tests.cpp +++ b/tests/src/metadata_tests.cpp @@ -5,11 +5,16 @@ namespace databento::tests { TEST(MetadataTests, TestDatasetConditionDetailToString) { - const DatasetConditionDetail target{"2022-11-10", DatasetCondition::Available, - "2023-03-01"}; - ASSERT_EQ( - ToString(target), + const DatasetConditionDetail available{ + "2022-11-10", DatasetCondition::Available, "2023-03-01"}; + EXPECT_EQ( + ToString(available), R"(DatasetConditionDetail { date = "2022-11-10", condition = available, last_modified_date = "2023-03-01" })"); + const DatasetConditionDetail missing{ + "2022-11-11", DatasetCondition::Missing, {}}; + EXPECT_EQ( + ToString(missing), + R"(DatasetConditionDetail { date = "2022-11-11", condition = missing, last_modified_date = nullopt })"); } TEST(MetadataTests, TestDatasetRangeToString) { From 6cd9ad9203661f39b1147fcfaf8dcb41d6503c78 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Tue, 24 Jun 2025 17:40:44 -0500 Subject: [PATCH 6/7] MOD: Use `POST` for `TimeseriesGetRange` methods --- CHANGELOG.md | 2 + include/databento/detail/http_client.hpp | 12 ++- include/databento/historical.hpp | 2 - src/detail/http_client.cpp | 77 +++++++++---- src/historical.cpp | 30 +++--- tests/include/mock/mock_http_server.hpp | 33 ++++-- tests/src/historical_tests.cpp | 88 +++++++-------- tests/src/mock_http_server.cpp | 131 ++++++++++++----------- 8 files changed, 221 insertions(+), 154 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f9a30a8..f941510 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,8 @@ ### Enhancements - Added `range_by_schema` field to `DatasetRange` struct +- Changed historical `TimeseriesGetRange` and `TimeseriesGetRangeToFile` methods to use + a `POST` request to allow for requesting supported maximum of 2000 symbols - Added logging around `Historical::BatchDownload` - Changed the following Venue, Publisher, and Dataset descriptions: - "ICE Futures Europe (Financials)" renamed to "ICE Europe Financials" diff --git a/include/databento/detail/http_client.hpp b/include/databento/detail/http_client.hpp index 74075db..2ac109e 100644 --- a/include/databento/detail/http_client.hpp +++ b/include/databento/detail/http_client.hpp @@ -24,15 +24,23 @@ class HttpClient { nlohmann::json GetJson(const std::string& path, const httplib::Params& params); nlohmann::json PostJson(const std::string& path, - const httplib::Params& params); + const httplib::Params& form_params); void GetRawStream(const std::string& path, const httplib::Params& params, const httplib::ContentReceiver& callback); + void PostRawStream(const std::string& path, + const httplib::Params& form_params, + const httplib::ContentReceiver& callback); private: + static bool IsErrorStatus(int status_code); + static httplib::ResponseHandler MakeStreamResponseHandler(int& out_status); + static void CheckStatusAndStreamRes(const std::string& path, int status_code, + std::string&& err_body, + const httplib::Result& res); + nlohmann::json CheckAndParseResponse(const std::string& path, httplib::Result&& res) const; void CheckWarnings(const httplib::Response& response) const; - static bool IsErrorStatus(int status_code); static const httplib::Headers kHeaders; diff --git a/include/databento/historical.hpp b/include/databento/historical.hpp index d526b94..8ca6720 100644 --- a/include/databento/historical.hpp +++ b/include/databento/historical.hpp @@ -230,8 +230,6 @@ class Historical { using HttplibParams = std::multimap; BatchJob BatchSubmitJob(const HttplibParams& params); - void StreamToFile(const std::string& url_path, const HttplibParams& params, - const std::filesystem::path& file_path); void DownloadFile(const std::string& url, const std::filesystem::path& output_path); std::vector BatchListJobs(const HttplibParams& params); diff --git a/src/detail/http_client.cpp b/src/detail/http_client.cpp index 45347c7..ba63f5d 100644 --- a/src/detail/http_client.cpp +++ b/src/detail/http_client.cpp @@ -53,25 +53,64 @@ void HttpClient::GetRawStream(const std::string& path, const std::string full_path = httplib::append_query_params(path, params); std::string err_body{}; int err_status{}; - const httplib::Result res = client_.Get( - full_path, - [&err_status](const httplib::Response& resp) { - if (HttpClient::IsErrorStatus(resp.status)) { - err_status = resp.status; - } - return true; - }, - [&callback, &err_body, &err_status](const char* data, - std::size_t length) { - // if an error response was received, read all content into err_status - if (err_status > 0) { - err_body.append(data, length); - return true; - } - return callback(data, length); - }); - if (err_status > 0) { - throw HttpResponseError{path, err_status, std::move(err_body)}; + const httplib::Result res = + client_.Get(full_path, MakeStreamResponseHandler(err_status), + [&callback, &err_body, &err_status](const char* data, + std::size_t length) { + // if an error response was received, read all content into + // err_body + if (err_status > 0) { + err_body.append(data, length); + return true; + } + return callback(data, length); + }); + CheckStatusAndStreamRes(path, err_status, std::move(err_body), res); +} + +void HttpClient::PostRawStream(const std::string& path, + const httplib::Params& form_params, + const httplib::ContentReceiver& callback) { + std::string err_body{}; + int err_status{}; + httplib::Request req; + req.method = "POST"; + req.set_header("Content-Type", "application/x-www-form-urlencoded"); + req.path = path; + req.body = httplib::detail::params_to_query_str(form_params); + req.response_handler = MakeStreamResponseHandler(err_status); + req.content_receiver = [&callback, &err_body, &err_status]( + const char* data, std::size_t length, + std::uint64_t, std::uint64_t) { + // if an error response was received, read all content into + // err_body + if (err_status > 0) { + err_body.append(data, length); + return true; + } + return callback(data, length); + }; + // NOLINTNEXTLINE(clang-analyzer-unix.BlockInCriticalSection): dependency code + const httplib::Result res = client_.send(req); + CheckStatusAndStreamRes(path, err_status, std::move(err_body), res); +} + +httplib::ResponseHandler HttpClient::MakeStreamResponseHandler( + int& out_status) { + return [&out_status](const httplib::Response& resp) { + if (HttpClient::IsErrorStatus(resp.status)) { + out_status = resp.status; + } + return true; + }; +} + +void HttpClient::CheckStatusAndStreamRes(const std::string& path, + int status_code, + std::string&& err_body, + const httplib::Result& res) { + if (status_code > 0) { + throw HttpResponseError{path, status_code, std::move(err_body)}; } if (res.error() != httplib::Error::Success && // canceled happens if `callback` returns false, which is based on the diff --git a/src/historical.cpp b/src/historical.cpp index 4eeaf65..5fdb0e2 100644 --- a/src/historical.cpp +++ b/src/historical.cpp @@ -316,17 +316,6 @@ std::filesystem::path Historical::BatchDownload( return output_path; } -void Historical::StreamToFile(const std::string& url_path, - const HttplibParams& params, - const std::filesystem::path& file_path) { - OutFileStream out_file{file_path}; - this->client_.GetRawStream( - url_path, params, [&out_file](const char* data, std::size_t length) { - out_file.WriteAll(reinterpret_cast(data), length); - return true; - }); -} - void Historical::DownloadFile(const std::string& url, const std::filesystem::path& output_path) { static const std::string kMethod = "Historical::DownloadFile"; @@ -352,7 +341,12 @@ void Historical::DownloadFile(const std::string& url, << output_path; log_receiver_->Receive(LogLevel::Info, ss.str()); - StreamToFile(path, {}, output_path); + OutFileStream out_file{output_path}; + this->client_.GetRawStream( + path, {}, [&out_file](const char* data, std::size_t length) { + out_file.WriteAll(reinterpret_cast(data), length); + return true; + }); if (log_receiver_->ShouldLog(LogLevel::Debug)) { ss.str(""); @@ -880,7 +874,7 @@ void Historical::TimeseriesGetRange(const HttplibParams& params, detail::DbnBufferDecoder decoder{metadata_callback, record_callback}; bool early_exit = false; - this->client_.GetRawStream( + this->client_.PostRawStream( kTimeseriesGetRangePath, params, [&decoder, &early_exit](const char* data, std::size_t length) mutable { if (decoder.Process(data, length) == KeepGoing::Continue) { @@ -959,7 +953,15 @@ databento::DbnFileStore Historical::TimeseriesGetRangeToFile( } databento::DbnFileStore Historical::TimeseriesGetRangeToFile( const HttplibParams& params, const std::filesystem::path& file_path) { - StreamToFile(kTimeseriesGetRangePath, params, file_path); + { + OutFileStream out_file{file_path}; + this->client_.PostRawStream( + kTimeseriesGetRangePath, params, + [&out_file](const char* data, std::size_t length) { + out_file.WriteAll(reinterpret_cast(data), length); + return true; + }); + } // Flush out_file return DbnFileStore{log_receiver_, file_path, VersionUpgradePolicy::UpgradeToV3}; } diff --git a/tests/include/mock/mock_http_server.hpp b/tests/include/mock/mock_http_server.hpp index 724127a..17cd368 100644 --- a/tests/include/mock/mock_http_server.hpp +++ b/tests/include/mock/mock_http_server.hpp @@ -5,8 +5,10 @@ #include #include +#include #include +#include "databento/detail/buffer.hpp" #include "databento/detail/scoped_thread.hpp" #include "databento/record.hpp" @@ -23,7 +25,7 @@ class MockHttpServer { ~MockHttpServer() { server_.stop(); } int ListenOnThread(); - void MockBadRequest(const std::string& path, const nlohmann::json& json); + void MockBadPostRequest(const std::string& path, const nlohmann::json& json); void MockGetJson(const std::string& path, const nlohmann::json& json); void MockGetJson(const std::string& path, const std::map& params, @@ -34,22 +36,31 @@ class MockHttpServer { void MockPostJson(const std::string& path, const std::map& params, const nlohmann::json& json); - void MockStreamDbn(const std::string& path, - const std::map& params, - const std::string& dbn_path); - void MockStreamDbn(const std::string& path, - const std::map& params, - Record record, std::size_t count, std::size_t chunk_size); - void MockStreamDbn(const std::string& path, - const std::map& params, - Record record, std::size_t count, std::size_t extra_bytes, - std::size_t chunk_size); + void MockGetDbn(const std::string& path, + const std::map& params, + const std::string& dbn_path); + void MockPostDbn(const std::string& path, + const std::map& params, + const std::string& dbn_path); + void MockPostDbn(const std::string& path, + const std::map& params, + Record record, std::size_t count, std::size_t chunk_size); + void MockPostDbn(const std::string& path, + const std::map& params, + Record record, std::size_t count, std::size_t extra_bytes, + std::size_t chunk_size); private: + using SharedConstBuffer = std::shared_ptr; + static void CheckParams(const std::map& params, const httplib::Request& req); static void CheckFormParams(const std::map& params, const httplib::Request& req); + static SharedConstBuffer EncodeToBuffer(const std::string& dbn_path); + static httplib::Server::Handler MakeDbnStreamHandler( + const std::map& params, + SharedConstBuffer&& buffer, std::size_t chunk_size); httplib::Server server_{}; const int port_{}; diff --git a/tests/src/historical_tests.cpp b/tests/src/historical_tests.cpp index 3e201b5..529783d 100644 --- a/tests/src/historical_tests.cpp +++ b/tests/src/historical_tests.cpp @@ -241,8 +241,8 @@ TEST_F(HistoricalTests, TestBatchDownloadAll) { const TempFile temp_dbn_file{tmp_path_ / "job123/test.dbn"}; mock_server_.MockGetJson("/v0/batch.list_files", {{"job_id", kJobId}}, kListFilesResp); - mock_server_.MockStreamDbn("/v0/job_id/test.dbn", {}, - TEST_DATA_DIR "/test_data.mbo.v3.dbn"); + mock_server_.MockGetDbn("/v0/job_id/test.dbn", {}, + TEST_DATA_DIR "/test_data.mbo.v3.dbn"); mock_server_.MockGetJson("/v0/job_id/test_metadata.json", {{"key", "value"}}); const auto port = mock_server_.ListenOnThread(); @@ -640,17 +640,17 @@ TEST_F(HistoricalTests, TestSymbologyResolve) { } TEST_F(HistoricalTests, TestTimeseriesGetRange_Basic) { - mock_server_.MockStreamDbn("/v0/timeseries.get_range", - {{"dataset", dataset::kGlbxMdp3}, - {"symbols", "ESH1"}, - {"schema", "mbo"}, - {"start", "1609160400000711344"}, - {"end", "1609160800000711344"}, - {"encoding", "dbn"}, - {"stype_in", "raw_symbol"}, - {"stype_out", "instrument_id"}, - {"limit", "2"}}, - TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst"); + mock_server_.MockPostDbn("/v0/timeseries.get_range", + {{"dataset", dataset::kGlbxMdp3}, + {"symbols", "ESH1"}, + {"schema", "mbo"}, + {"start", "1609160400000711344"}, + {"end", "1609160800000711344"}, + {"encoding", "dbn"}, + {"stype_in", "raw_symbol"}, + {"stype_out", "instrument_id"}, + {"limit", "2"}}, + TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst"); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", @@ -676,16 +676,16 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_Basic) { } TEST_F(HistoricalTests, TestTimeseriesGetRange_NoMetadataCallback) { - mock_server_.MockStreamDbn("/v0/timeseries.get_range", - {{"dataset", dataset::kGlbxMdp3}, - {"start", "2022-10-21T13:30"}, - {"end", "2022-10-21T20:00"}, - {"symbols", "CYZ2"}, - {"schema", "tbbo"}, - {"encoding", "dbn"}, - {"stype_in", "raw_symbol"}, - {"stype_out", "instrument_id"}}, - TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst"); + mock_server_.MockPostDbn("/v0/timeseries.get_range", + {{"dataset", dataset::kGlbxMdp3}, + {"start", "2022-10-21T13:30"}, + {"end", "2022-10-21T20:00"}, + {"symbols", "CYZ2"}, + {"schema", "tbbo"}, + {"encoding", "dbn"}, + {"stype_in", "raw_symbol"}, + {"stype_out", "instrument_id"}}, + TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst"); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", @@ -704,7 +704,7 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_NoMetadataCallback) { TEST_F(HistoricalTests, TestTimeseriesGetRange_BadRequest) { const nlohmann::json resp{ {"detail", "Authorization failed: illegal chars in username."}}; - mock_server_.MockBadRequest("/v0/timeseries.get_range", resp); + mock_server_.MockBadPostRequest("/v0/timeseries.get_range", resp); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", @@ -729,8 +729,8 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_BadRequest) { } TEST_F(HistoricalTests, TestTimeseriesGetRange_CallbackException) { - mock_server_.MockStreamDbn("/v0/timeseries.get_range", {}, - TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst"); + mock_server_.MockPostDbn("/v0/timeseries.get_range", {}, + TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst"); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", @@ -747,8 +747,8 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_CallbackException) { } TEST_F(HistoricalTests, TestTimeseriesGetRange_Cancellation) { - mock_server_.MockStreamDbn("/v0/timeseries.get_range", {}, - TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst"); + mock_server_.MockPostDbn("/v0/timeseries.get_range", {}, + TEST_DATA_DIR "/test_data.mbo.v3.dbn.zst"); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", @@ -777,9 +777,9 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_LargeChunks) { 10005, {}}}; constexpr auto kRecordCount = 50'000; - mock_server_.MockStreamDbn("/v0/timeseries.get_range", - {{"dataset", ToString(Dataset::IfusImpact)}}, - Record{&mbp1.hd}, kRecordCount, 75'000); + mock_server_.MockPostDbn("/v0/timeseries.get_range", + {{"dataset", ToString(Dataset::IfusImpact)}}, + Record{&mbp1.hd}, kRecordCount, 75'000); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", @@ -804,9 +804,9 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_UnreadBytes) { 10005, {}}}; constexpr auto kRecordCount = 1'000; - mock_server_.MockStreamDbn("/v0/timeseries.get_range", - {{"dataset", ToString(Dataset::IfusImpact)}}, - Record{&mbp1.hd}, kRecordCount, 20, 75'000); + mock_server_.MockPostDbn("/v0/timeseries.get_range", + {{"dataset", ToString(Dataset::IfusImpact)}}, + Record{&mbp1.hd}, kRecordCount, 20, 75'000); const auto port = mock_server_.ListenOnThread(); logger_ = mock::MockLogReceiver{[](auto count, LogLevel level, @@ -830,16 +830,16 @@ TEST_F(HistoricalTests, TestTimeseriesGetRange_UnreadBytes) { } TEST_F(HistoricalTests, TestTimeseriesGetRangeToFile) { - mock_server_.MockStreamDbn("/v0/timeseries.get_range", - {{"dataset", dataset::kGlbxMdp3}, - {"start", "2022-10-21T13:30"}, - {"end", "2022-10-21T20:00"}, - {"symbols", "CYZ2"}, - {"schema", "tbbo"}, - {"encoding", "dbn"}, - {"stype_in", "raw_symbol"}, - {"stype_out", "instrument_id"}}, - TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst"); + mock_server_.MockPostDbn("/v0/timeseries.get_range", + {{"dataset", dataset::kGlbxMdp3}, + {"start", "2022-10-21T13:30"}, + {"end", "2022-10-21T20:00"}, + {"symbols", "CYZ2"}, + {"schema", "tbbo"}, + {"encoding", "dbn"}, + {"stype_in", "raw_symbol"}, + {"stype_out", "instrument_id"}}, + TEST_DATA_DIR "/test_data.tbbo.v3.dbn.zst"); const auto port = mock_server_.ListenOnThread(); databento::Historical target{&logger_, kApiKey, "localhost", diff --git a/tests/src/mock_http_server.cpp b/tests/src/mock_http_server.cpp index d5dc69b..e8cde50 100644 --- a/tests/src/mock_http_server.cpp +++ b/tests/src/mock_http_server.cpp @@ -4,15 +4,14 @@ #include #include -#include // ifstream -#include // streamsize -#include // cerr #include #include "databento/constants.hpp" #include "databento/dbn.hpp" #include "databento/dbn_encoder.hpp" +#include "databento/detail/buffer.hpp" #include "databento/detail/zstd_stream.hpp" +#include "databento/file_stream.hpp" #include "databento/record.hpp" using databento::tests::mock::MockHttpServer; @@ -23,9 +22,9 @@ int MockHttpServer::ListenOnThread() { return port_; } -void MockHttpServer::MockBadRequest(const std::string& path, - const nlohmann::json& json) { - server_.Get(path, [json](const httplib::Request&, httplib::Response& resp) { +void MockHttpServer::MockBadPostRequest(const std::string& path, + const nlohmann::json& json) { + server_.Post(path, [json](const httplib::Request&, httplib::Response& resp) { resp.status = 400; resp.body = json.dump(); }); @@ -78,50 +77,39 @@ void MockHttpServer::MockPostJson( }); } -void MockHttpServer::MockStreamDbn( +void MockHttpServer::MockGetDbn( const std::string& path, const std::map& params, const std::string& dbn_path) { constexpr std::size_t kChunkSize = 32; // Read contents into buffer - std::ifstream input_file{dbn_path, std::ios::binary | std::ios::ate}; - const auto size = static_cast(input_file.tellg()); - input_file.seekg(0, std::ios::beg); - std::vector buffer(size); - input_file.read(buffer.data(), static_cast(size)); + auto buffer = EncodeToBuffer(dbn_path); // Serve - server_.Get(path, [buffer = std::move(buffer), kChunkSize, params]( - const httplib::Request& req, httplib::Response& resp) { - if (!req.has_header("Authorization")) { - resp.status = 401; - return; - } - CheckParams(params, req); - resp.status = 200; - resp.set_header("Content-Disposition", "attachment; filename=test.dbn.zst"); - resp.set_content_provider( - "application/octet-stream", - [buffer, kChunkSize](const std::size_t offset, - httplib::DataSink& sink) { - if (offset < buffer.size()) { - sink.write(&buffer[offset], - std::min(kChunkSize, buffer.size() - offset)); - } else { - sink.done(); - } - return true; - }); - }); + server_.Get(path, + MakeDbnStreamHandler(params, std::move(buffer), kChunkSize)); +} + +void MockHttpServer::MockPostDbn( + const std::string& path, const std::map& params, + const std::string& dbn_path) { + constexpr std::size_t kChunkSize = 32; + + // Read contents into buffer + auto buffer = EncodeToBuffer(dbn_path); + + // Serve + server_.Post(path, + MakeDbnStreamHandler(params, std::move(buffer), kChunkSize)); } -void MockHttpServer::MockStreamDbn( +void MockHttpServer::MockPostDbn( const std::string& path, const std::map& params, Record record, std::size_t count, std::size_t chunk_size) { - MockStreamDbn(path, params, record, count, 0, chunk_size); + MockPostDbn(path, params, record, count, 0, chunk_size); } -void MockHttpServer::MockStreamDbn( +void MockHttpServer::MockPostDbn( const std::string& path, const std::map& params, Record record, std::size_t count, std::size_t extra_bytes, std::size_t chunk_size) { @@ -146,31 +134,8 @@ void MockHttpServer::MockStreamDbn( zstd_stream.WriteAll(empty.data(), empty.size()); } } - server_.Get(path, [params, buffer, count, record, chunk_size]( - const httplib::Request& req, httplib::Response& resp) { - if (!req.has_header("Authorization")) { - resp.status = 401; - return; - } - CheckParams(params, req); - resp.status = 200; - resp.set_header("Content-Disposition", "attachment; filename=test.dbn.zst"); - resp.set_content_provider( - "application/octet-stream", - [&buffer, chunk_size](const std::size_t offset, - httplib::DataSink& sink) { - if (buffer->ReadCapacity() - offset) { - const auto write_size = - std::min(chunk_size, buffer->ReadCapacity() - offset); - sink.write( - reinterpret_cast(&buffer->ReadBegin()[offset]), - write_size); - } else { - sink.done(); - } - return true; - }); - }); + server_.Post(path, + MakeDbnStreamHandler(params, std::move(buffer), chunk_size)); } void MockHttpServer::CheckParams( @@ -205,3 +170,45 @@ void MockHttpServer::CheckFormParams( } } } + +MockHttpServer::SharedConstBuffer MockHttpServer::EncodeToBuffer( + const std::string& dbn_path) { + detail::Buffer buffer{}; + InFileStream input_file{dbn_path}; + while (auto read_size = + input_file.ReadSome(buffer.WriteBegin(), buffer.WriteCapacity())) { + buffer.Fill(read_size); + if (buffer.WriteCapacity() < 1024) { + buffer.Reserve(buffer.Capacity() * 2); + } + } + return std::make_shared(std::move(buffer)); +} + +httplib::Server::Handler MockHttpServer::MakeDbnStreamHandler( + const std::map& params, + SharedConstBuffer&& buffer, std::size_t chunk_size) { + return [buffer = std::move(buffer), chunk_size, params]( + const httplib::Request& req, httplib::Response& resp) { + if (!req.has_header("Authorization")) { + resp.status = 401; + return; + } + CheckParams(params, req); + resp.status = 200; + resp.set_header("Content-Disposition", "attachment; filename=test.dbn.zst"); + resp.set_content_provider( + "application/octet-stream", + [buffer, chunk_size](const std::size_t offset, + httplib::DataSink& sink) { + if (offset < buffer->ReadCapacity()) { + sink.write( + reinterpret_cast(&buffer->ReadBegin()[offset]), + std::min(chunk_size, buffer->ReadCapacity() - offset)); + } else { + sink.done(); + } + return true; + }); + }; +} From 93e53b706a888cda3b4b11b2c26b1e7810e21f07 Mon Sep 17 00:00:00 2001 From: Carter Green Date: Wed, 25 Jun 2025 09:24:20 -0500 Subject: [PATCH 7/7] VER: Release 0.38.1 --- CHANGELOG.md | 2 +- CMakeLists.txt | 2 +- include/databento/metadata.hpp | 2 +- pkg/PKGBUILD | 2 +- src/historical.cpp | 3 +-- 5 files changed, 5 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f941510..4208fea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # Changelog -## 0.38.1 - TBD +## 0.38.1 - 2025-06-25 ### Enhancements - Added `range_by_schema` field to `DatasetRange` struct diff --git a/CMakeLists.txt b/CMakeLists.txt index 264eeda..a9b1264 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -6,7 +6,7 @@ cmake_minimum_required(VERSION 3.24..4.0) project( databento - VERSION 0.38.0 + VERSION 0.38.1 LANGUAGES CXX DESCRIPTION "Official Databento client library" ) diff --git a/include/databento/metadata.hpp b/include/databento/metadata.hpp index 54575a0..6fb3c2d 100644 --- a/include/databento/metadata.hpp +++ b/include/databento/metadata.hpp @@ -37,7 +37,7 @@ struct DatasetConditionDetail { struct DatasetRange { std::string start; std::string end; - std::unordered_map> range_by_schema; + std::map> range_by_schema; }; inline bool operator==(const PublisherDetail& lhs, const PublisherDetail& rhs) { diff --git a/pkg/PKGBUILD b/pkg/PKGBUILD index b57d262..013976e 100644 --- a/pkg/PKGBUILD +++ b/pkg/PKGBUILD @@ -1,7 +1,7 @@ # Maintainer: Databento _pkgname=databento-cpp pkgname=databento-cpp-git -pkgver=0.38.0 +pkgver=0.38.1 pkgrel=1 pkgdesc="Official C++ client for Databento" arch=('any') diff --git a/src/historical.cpp b/src/historical.cpp index 5fdb0e2..ec2b1a5 100644 --- a/src/historical.cpp +++ b/src/historical.cpp @@ -14,7 +14,6 @@ #include #include #include -#include #include // move #include "databento/constants.hpp" @@ -537,7 +536,7 @@ databento::DatasetRange Historical::MetadataGetDatasetRange( if (!schema_json.is_object()) { throw JsonResponseError::TypeMismatch(kEndpoint, "schema object", json); } - std::unordered_map> range_by_schema; + std::map> range_by_schema; for (const auto& schema_item : schema_json.items()) { if (!schema_item.value().is_object()) { throw JsonResponseError::TypeMismatch(kEndpoint, "nested schema object",