36 live websocket data streaming and storage

.github/workflows/ci.yml

@@ -19,6 +19,12 @@ jobs:
         sudo apt-get update
         sudo apt-get install -y g++ cmake make
 
+    - name: Install dependencies
+      run: |
+          sudo apt-get update
+          sudo apt-get install -y libcurl4-openssl-dev
+          sudo apt-get install -y libboost-all-dev
+
     - name: Verify G++ version
       run: g++ --version
 

CMakeLists.txt

@@ -1,10 +1,15 @@
+# Copyright (c) 2025 arvindkrv@protonmail.com
+#
+# Please see the LICENSE file for the terms and conditions
+# associated with this software.
+
 cmake_minimum_required(VERSION 3.20)
 project(cryptoquantengine LANGUAGES CXX)
 
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-add_executable(hftbacktest
+add_executable(backtest
   cryptoquantengine/backtest_main.cpp
   cryptoquantengine/core/orderbook/orderbook.cpp
   cryptoquantengine/utils/config/config_reader.cpp
@@ -19,19 +24,40 @@ add_executable(hftbacktest
   cryptoquantengine/utils/logger/logger.cpp
 )
 
-target_include_directories(hftbacktest PRIVATE
+target_include_directories(backtest PRIVATE
   ${CMAKE_CURRENT_SOURCE_DIR}/cryptoquantengine
   ${CMAKE_CURRENT_SOURCE_DIR}/external 
+  ${CMAKE_CURRENT_SOURCE_DIR}/external/websocketpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/external/json
 )
 
-target_compile_options(hftbacktest PRIVATE
+target_compile_options(backtest PRIVATE
   $<$<CONFIG:Release>:-O3>
   $<$<CONFIG:Debug>:-O3 -Wall -Wextra -Wpedantic>
 )
 
-target_link_libraries(hftbacktest PRIVATE Threads::Threads)
+target_link_libraries(backtest PRIVATE Threads::Threads)
+
+add_executable(stream
+  cryptoquantengine/wss_main.cc
+  cryptoquantengine/core/market_data/readers/ws/binance_stream_reader.cc
+  cryptoquantengine/core/market_data/readers/ws/websocket_stream_reader.cc
+)
+
+set_target_properties(stream PROPERTIES CXX_STANDARD 17 CXX_STANDARD_REQUIRED YES)
+
+target_include_directories(stream PRIVATE
+  ${CMAKE_CURRENT_SOURCE_DIR}/cryptoquantengine
+  ${CMAKE_CURRENT_SOURCE_DIR}/external
+  ${CMAKE_CURRENT_SOURCE_DIR}/external/websocketpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/external/json
+)
+
+target_compile_options(stream PRIVATE
+  $<$<CONFIG:Release>:-O3>
+  $<$<CONFIG:Debug>:-O3 -Wall -Wextra -Wpedantic>
+)
 
-# Catch2 test setup
 include(FetchContent)
 FetchContent_Declare(
   Catch2
@@ -46,46 +72,58 @@ include(CTest)
 
 set(THREADS_PREFER_PTHREAD_FLAG ON)
 find_package(Threads REQUIRED)
+find_package(OpenSSL REQUIRED)
+find_package(CURL REQUIRED)
 
-function(add_hft_test_executable target_name source_files)
+target_link_libraries(stream PRIVATE Threads::Threads)
+target_link_libraries(stream PRIVATE OpenSSL::SSL OpenSSL::Crypto)
+target_link_libraries(stream PRIVATE CURL::libcurl)
+
+function(add_test_executable target_name source_files)
   add_executable(${target_name} ${source_files})
-  target_include_directories(${target_name} PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/cryptoquantengine)
-  target_link_libraries(${target_name} PRIVATE Catch2::Catch2WithMain Threads::Threads)
+  target_include_directories(${target_name} PRIVATE 
+  ${CMAKE_CURRENT_SOURCE_DIR}/cryptoquantengine
+  ${CMAKE_CURRENT_SOURCE_DIR}/external
+  ${CMAKE_CURRENT_SOURCE_DIR}/external/websocketpp
+  ${CMAKE_CURRENT_SOURCE_DIR}/external/json)
+  target_link_libraries(${target_name} PRIVATE Catch2::Catch2WithMain Threads::Threads OpenSSL::SSL OpenSSL::Crypto CURL::libcurl)
+
   catch_discover_tests(${target_name})
 endfunction()
 
-add_hft_test_executable(test_orderbook 
+add_test_executable(test_orderbook 
   "tests/core/test_orderbook.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable(test_config_reader 
+add_test_executable(test_config_reader 
   "tests/utils/test_config_reader.cpp;cryptoquantengine/utils/config/config_reader.cpp"
 )
-add_hft_test_executable(test_bookstream_reader
+add_test_executable(test_bookstream_reader
   "tests/market_data/test_book_stream_reader.cpp;cryptoquantengine/core/market_data/readers/book_stream_reader.cpp;cryptoquantengine/core/market_data/readers/base_stream_reader.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable(test_trade_stream_reader
+add_test_executable(test_trade_stream_reader
   "tests/market_data/test_trade_stream_reader.cpp;cryptoquantengine/core/market_data/readers/trade_stream_reader.cpp;cryptoquantengine/core/market_data/readers/base_stream_reader.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable(test_market_data_feed
+add_test_executable(test_market_data_feed
   "tests/market_data/test_market_data_feed.cpp;cryptoquantengine/core/market_data/market_data_feed.cpp;cryptoquantengine/core/market_data/readers/base_stream_reader.cpp;cryptoquantengine/core/market_data/readers/trade_stream_reader.cpp;cryptoquantengine/core/market_data/readers/book_stream_reader.cpp"	
 )
-add_hft_test_executable(test_execution_engine "tests/core/test_execution_engine.cpp;cryptoquantengine/core/execution_engine/execution_engine.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/utils/logger/logger.cpp"
+add_test_executable(test_execution_engine "tests/core/test_execution_engine.cpp;cryptoquantengine/core/execution_engine/execution_engine.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable(test_backtest_engine 
+add_test_executable(test_backtest_engine 
   "tests/core/test_backtest_engine.cpp;cryptoquantengine/core/backtest_engine/backtest_engine.cpp;cryptoquantengine/core/execution_engine/execution_engine.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/core/market_data/market_data_feed.cpp;cryptoquantengine/core/market_data/readers/base_stream_reader.cpp;cryptoquantengine/core/market_data/readers/trade_stream_reader.cpp;cryptoquantengine/core/market_data/readers/book_stream_reader.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable(test_stat_utils 
+add_test_executable(test_stat_utils 
   "tests/utils/test_stat_utils.cpp"
 )
-add_hft_test_executable(test_recorder 
+add_test_executable(test_recorder 
   "tests/core/test_recorder.cpp;cryptoquantengine/core/recorder/recorder.cpp;cryptoquantengine/core/backtest_engine/backtest_engine.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/core/execution_engine/execution_engine.cpp;cryptoquantengine/core/market_data/market_data_feed.cpp;cryptoquantengine/core/market_data/readers/base_stream_reader.cpp;cryptoquantengine/core/market_data/readers/trade_stream_reader.cpp;cryptoquantengine/core/market_data/readers/book_stream_reader.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable (test_grid_trading 
+add_test_executable (test_grid_trading 
   "tests/strategies/test_grid_trading.cpp;cryptoquantengine/core/strategy/grid_trading/grid_trading.cpp;cryptoquantengine/core/execution_engine/execution_engine.cpp;cryptoquantengine/core/backtest_engine/backtest_engine.cpp;cryptoquantengine/core/orderbook/orderbook.cpp;cryptoquantengine/core/market_data/market_data_feed.cpp;cryptoquantengine/core/market_data/readers/base_stream_reader.cpp;cryptoquantengine/core/market_data/readers/trade_stream_reader.cpp;cryptoquantengine/core/market_data/readers/book_stream_reader.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
-add_hft_test_executable (test_math_utils 
+add_test_executable (test_math_utils 
   "tests/utils/test_math_utils.cpp"
 )
-add_hft_test_executable (test_logger 
+add_test_executable (test_logger 
   "tests/utils/test_logger.cpp;cryptoquantengine/utils/logger/logger.cpp"
 )
+#add_test_executable (test_binance_stream "tests/market_data/live/test_binance_stream_reader.cpp;cryptoquantengine/core/market_data/readers/ws/binance_stream_reader.cc;cryptoquantengine/core/market_data/readers/ws/websocket_stream_reader.cc")

README.md

@@ -1,6 +1,7 @@
 # CryptoQuantEngine
 
-<img src="https://github.com/caped-doshi/cryptoquantengine/actions/workflows/ci.yml/badge.svg"> [![GitHub release](https://img.shields.io/github/v/release/caped-doshi/cryptoquantengine)](https://github.com/caped-doshi/cryptoquantengine/releases) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
+<img src="https://github.com/caped-doshi/cryptoquantengine/actions/workflows/ci.yml/badge.svg"> [![Python version](https://img.shields.io/badge/python-3.7+-blue)](https://www.python.org/downloads/release/python-370/)
+ [![GitHub release](https://img.shields.io/github/v/release/caped-doshi/cryptoquantengine)](https://github.com/caped-doshi/cryptoquantengine/releases) [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
 
 ## High-Frequency Level 2 Trading Backtesting Engine
 
@@ -64,7 +65,7 @@ cmake -DCMAKE_BUILD_TYPE=Release ..
 
 Compile the engine
 ```bash
-cmake --build . --config Release
+cmake --build .
 ```
 
 Run unit tests
@@ -74,7 +75,7 @@ ctest --output-on-failure
 
 Run the backtest engine
 ```bash
-./hftbacktest
+./backtest
 ```
 
 ---