VER: Release 0.20.0

CHANGELOG.md

@@ -1,5 +1,36 @@
 # Changelog
 
+## 0.20.0 - 2024-07-09
+
+This release adds support for encoding DBN within the C++ client.
+It also improves historical symbology support with the new `TsSymbolMap` class that
+handles mapping historical records to a text symbol. To support this class, several types
+for date fields were changed from strings or ints to `date::year_month_day`.
+
+### Enhancements
+- Added `TsSymbolMap` to support historical symbology where mappings change between days
+- Added `DbnEncoder` class for encoding DBN data
+- Added blocking API similar to `LiveBlocking` to `DbnFileStore` with new `GetMetadata`
+  and `NextRecord` methods
+- Added `BboMsg` record struct for future `bbo-1m` and `bbo-1s` schemas
+- Added `PitSymbol` map constructor from `Metadata` and a `date::year_month_day`
+- Added `Metadata::CreateSymbolMap` and `Metadata::CreateSymbolMapForDate` methods for
+  creating symbology maps from historical metadata
+- Added blocking API similar to `LiveBlocking` to `DbnFileStore`
+- Added `SymbologyResolution::CreateSymbolMap` method for creating a symbology map from
+  a symbology resolution response
+- Added `InFileStream` and `OutFileStream` helper classes for reading and writing binary
+  output respectively
+
+### Breaking changes
+- Added new dependency on [Howard Hinnant's date library](https://howardhinnant.github.io/date/date.html)
+- Added `ILogReceiver*` parameter to all `DbnDecoder` constructors and one `DbnFileStore` constructor
+- Removed type `StrMappingInterval`. `MappingInterval` is now also used in `SymbologyResolution`.
+- Changed `Bbo1sMsg` and `Bbo1mMsg` to be aliases for `BboMsg`
+- Changed type of `start_date` and `end_date` in `MappingInterval` to `date::year_month_day`
+- Added `stype_in` and `stype_out` fields to `SymbologyResolution` to support creating
+  a `TsSymbolMap`
+
 ## 0.19.1 - 2024-06-25
 
 ### Enhancements

CMakeLists.txt

@@ -4,7 +4,7 @@ cmake_minimum_required(VERSION 3.14)
 # Project details
 #
 
-project("databento" VERSION 0.19.1 LANGUAGES CXX)
+project("databento" VERSION 0.20.0 LANGUAGES CXX)
 string(TOUPPER ${PROJECT_NAME} PROJECT_NAME_UPPERCASE)
 
 #
@@ -181,6 +181,38 @@ else()
   # Ignore compiler warnings in headers
   add_system_include_property(httplib)
 endif()
+# date
+if(${PROJECT_NAME_UPPERCASE}_USE_EXTERNAL_DATE)
+  # Check if date target already exists
+  if(TARGET date::date)
+    get_target_property(DATE_SOURCE_DIR date::date SOURCE_DIR)
+    message(STATUS "date::date already available as a target at ${DATE_SOURCE_DIR}")
+    get_target_property(DATE_INCLUDE_DIRS date::date INTERFACE_INCLUDE_DIRECTORIES)
+    if(DATE_INCLUDE_DIRS)
+      message(STATUS "date::date include directories: ${DATE_INCLUDE_DIRS}")
+    endif()
+  else()
+    find_package(date REQUIRED)
+  endif()
+else()
+  set(date_version 3.0.1)
+  if(CMAKE_VERSION VERSION_LESS 3.24)
+    FetchContent_Declare(
+      date_src
+      URL https://github.com/HowardHinnant/date/archive/refs/tags/v${date_version}.tar.gz
+    )
+  else()
+    # DOWNLOAD_EXTRACT_TIMESTAMP added in 3.24
+    FetchContent_Declare(
+      date_src
+      URL https://github.com/HowardHinnant/date/archive/refs/tags/v${date_version}.tar.gz
+      DOWNLOAD_EXTRACT_TIMESTAMP TRUE
+    )
+  endif()
+  FetchContent_MakeAvailable(date_src)
+  # Ignore compiler warnings in headers
+  add_system_include_property(date)
+endif()
 # openSSL
 find_package(OpenSSL REQUIRED)
 if(OPENSSL_FOUND)
@@ -204,6 +236,7 @@ endif()
 target_link_libraries(
   ${PROJECT_NAME}
   PUBLIC
+    date::date
     httplib::httplib
     nlohmann_json::nlohmann_json
     OpenSSL::Crypto

README.md

@@ -64,11 +64,12 @@ You'll need to ensure the following dependencies are installed:
 - [Zstandard (zstd)](https://github.com/facebook/zstd)
 - [nlohmann\_json (header-only)](https://github.com/nlohmann/json)
 - [cpp-httplib (header-only)](https://github.com/yhirose/cpp-httplib)
+- [date (header-only)](https://github.com/HowardHinnant/date)
 - [dirent (Windows-only, header-only)](https://github.com/tronkko/dirent)
 
-By default, cpp-httplib and nlohmann\_json are downloaded by CMake as part of the build process.
+By default, date, cpp-httplib and nlohmann\_json are downloaded by CMake as part of the build process.
 If you would like to use a local version of these libraries, enable the CMake flag
-`DATABENTO_ENABLE_EXTERNAL_HTTPLIB` or `DATABENTO_ENABLE_EXTERNAL_JSON`.
+`DATABENTO_ENABLE_EXTERNAL_DATE`, `DATABENTO_ENABLE_EXTERNAL_HTTPLIB`, or `DATABENTO_ENABLE_EXTERNAL_JSON` respectively.
 
 #### Ubuntu
 
@@ -126,26 +127,32 @@ To run this program, set the `DATABENTO_API_KEY` environment variable with an ac
 
 ### Historical
 
-Here is a simple program that fetches 10 minutes worth of historical trades for the entire CME Globex market:
+Here is a simple program that fetches 10 minutes worth of historical trades for two CME futures:
 
 ```cpp
-#include <databento/constants.hpp>
+#include <databento/dbn.hpp>
 #include <databento/historical.hpp>
+#include <databento/symbol_map.hpp>
 #include <iostream>
 
 using namespace databento;
 
 int main() {
-  auto client = HistoricalBuilder{}.SetKeyFromEnv().Build();
-  auto print_trades = [](const Record& record) {
+  auto client = HistoricalBuilder{}.SetKey("$YOUR_API_KEY").Build();
+  TsSymbolMap symbol_map;
+  auto decode_symbols = [&symbol_map](const Metadata& metadata) {
+    symbol_map = metadata.CreateSymbolMap();
+  };
+  auto print_trades = [&symbol_map](const Record& record) {
     const auto& trade_msg = record.Get<TradeMsg>();
-    std::cout << trade_msg << '\n';
+    std::cout << "Received trade for " << symbol_map.At(trade_msg) << ": "
+              << trade_msg << '\n';
     return KeepGoing::Continue;
   };
-  client.TimeseriesGetRange("GLBX.MDP3",
-                            {"2022-06-10T14:30", "2022-06-10T14:40"},
-                            kAllSymbols, Schema::Trades, SType::RawSymbol,
-                            SType::InstrumentId, {}, {}, print_trades);
+  client.TimeseriesGetRange(
+      "GLBX.MDP3", {"2022-06-10T14:30", "2022-06-10T14:40"}, kAllSymbols,
+      Schema::Trades, SType::RawSymbol, SType::InstrumentId, {}, decode_symbols,
+      print_trades);
 }
 ```
 
@@ -156,7 +163,7 @@ These examples can be compiled by enabling the cmake option `DATABENTO_ENABLE_EX
 
 ## Documentation
 
-You can find more detailed examples and the full API documentation on the [Databento doc site](https://databento.com/docs/getting-started?historical=cpp&live=cpp).
+You can find more detailed examples and the full API documentation on the [Databento doc site](https://databento.com/docs/quickstart?historical=cpp&live=cpp).
 
 ## License
 

cmake/SourcesAndHeaders.cmake

@@ -5,8 +5,8 @@ set(headers
   include/databento/datetime.hpp
   include/databento/dbn.hpp
   include/databento/dbn_decoder.hpp
+  include/databento/dbn_encoder.hpp
   include/databento/dbn_file_store.hpp
-  include/databento/detail/file_stream.hpp
   include/databento/detail/http_client.hpp
   include/databento/detail/json_helpers.hpp
   include/databento/detail/scoped_fd.hpp
@@ -16,6 +16,7 @@ set(headers
   include/databento/detail/zstd_stream.hpp
   include/databento/enums.hpp
   include/databento/exceptions.hpp
+  include/databento/file_stream.hpp
   include/databento/fixed_price.hpp
   include/databento/flag_set.hpp
   include/databento/historical.hpp
@@ -39,9 +40,10 @@ set(sources
   src/compat.cpp
   src/datetime.cpp
   src/dbn.cpp
+  src/dbn_constants.hpp
   src/dbn_decoder.cpp
+  src/dbn_encoder.cpp
   src/dbn_file_store.cpp
-  src/detail/file_stream.cpp
   src/detail/http_client.cpp
   src/detail/json_helpers.cpp
   src/detail/scoped_fd.cpp
@@ -50,6 +52,7 @@ set(sources
   src/detail/zstd_stream.cpp
   src/enums.cpp
   src/exceptions.cpp
+  src/file_stream.cpp
   src/fixed_price.cpp
   src/flag_set.cpp
   src/historical.cpp

example/historical/readme.cpp

@@ -1,22 +1,28 @@
 // Duplicate of the example usage code from the README.md to ensure
 // it compiles and to be able to clang-format it.
 // NOLINTBEGIN(google-build-using-namespace)
-#include <databento/constants.hpp>
+#include <databento/dbn.hpp>
 #include <databento/historical.hpp>
+#include <databento/symbol_map.hpp>
 #include <iostream>
 
 using namespace databento;
 
 int main() {
   auto client = HistoricalBuilder{}.SetKey("$YOUR_API_KEY").Build();
-  auto print_trades = [](const Record& record) {
+  TsSymbolMap symbol_map;
+  auto decode_symbols = [&symbol_map](const Metadata& metadata) {
+    symbol_map = metadata.CreateSymbolMap();
+  };
+  auto print_trades = [&symbol_map](const Record& record) {
     const auto& trade_msg = record.Get<TradeMsg>();
-    std::cout << trade_msg << '\n';
+    std::cout << "Received trade for " << symbol_map.At(trade_msg) << ": "
+              << trade_msg << '\n';
     return KeepGoing::Continue;
   };
-  client.TimeseriesGetRange("GLBX.MDP3",
-                            {"2022-06-10T14:30", "2022-06-10T14:40"},
-                            kAllSymbols, Schema::Trades, SType::RawSymbol,
-                            SType::InstrumentId, {}, {}, print_trades);
+  client.TimeseriesGetRange(
+      "GLBX.MDP3", {"2022-06-10T14:30", "2022-06-10T14:40"}, {"ESM2", "NQZ2"},
+      Schema::Trades, SType::RawSymbol, SType::InstrumentId, {}, decode_symbols,
+      print_trades);
 }
 // NOLINTEND(google-build-using-namespace)

example/historical/symbology_resolve.cpp

@@ -16,7 +16,7 @@ int main(int argc, char* argv[]) {
   const auto stype_out = databento::FromString<databento::SType>(argv[3]);
 
   std::vector<std::string> symbols;
-  for (int i = 6; i < argc; ++i) {
+  for (int i = 5; i < argc; ++i) {
     symbols.emplace_back(argv[i]);
   }
 

include/databento/dbn.hpp

@@ -1,5 +1,7 @@
 #pragma once
 
+#include <date/date.h>
+
 #include <cstdint>
 #include <ostream>
 #include <string>
@@ -9,13 +11,15 @@
 #include "databento/enums.hpp"
 
 namespace databento {
+// Forward declare
+class PitSymbolMap;
+class TsSymbolMap;
+
 struct MappingInterval {
-  // The start date of the interval (inclusive) as
-  // YYYYMMDD e.g. 2022-10-08 is represented as 20221008
-  std::uint32_t start_date;
-  // The end date of the interval (exclusive) as
-  // YYYYMMDD e.g. 2022-10-08 is represented as 20221008
-  std::uint32_t end_date;
+  // The start date of the interval (inclusive).
+  date::year_month_day start_date;
+  // The end date of the interval (exclusive).
+  date::year_month_day end_date;
   std::string symbol;
 };
 
@@ -69,6 +73,16 @@ struct Metadata {
   // Symbol mappings containing a native symbol and its mapping intervals.
   std::vector<SymbolMapping> mappings;
 
+  // Creates a symbology mapping from instrument ID to text symbol for the given
+  // date.
+  //
+  // This method is useful when working with a historical request over a single
+  // day or in other situations where you're sure the mappings don't change
+  // during the time range of the request. Otherwise, `SymbolMap()` is
+  // recommmended.
+  PitSymbolMap CreateSymbolMapForDate(date::year_month_day date) const;
+  // Creates a symbology mapping from instrument ID and date to text symbol.
+  TsSymbolMap CreateSymbolMap() const;
   // Upgrades the metadata according to `upgrade_policy` if necessary.
   void Upgrade(VersionUpgradePolicy upgrade_policy);
 };

include/databento/dbn_decoder.hpp

@@ -6,25 +6,24 @@
 #include <string>
 
 #include "databento/dbn.hpp"
-#include "databento/detail/file_stream.hpp"
 #include "databento/detail/shared_channel.hpp"
 #include "databento/enums.hpp"  // Upgrade Policy
+#include "databento/file_stream.hpp"
 #include "databento/ireadable.hpp"
+#include "databento/log.hpp"
 #include "databento/record.hpp"  // Record, RecordHeader
 
 namespace databento {
 // DBN decoder. Set upgrade_policy to control how DBN version 1 data should be
 // handled. Defaults to upgrading DBNv1 data to version 2 (the current version).
 class DbnDecoder {
  public:
-  explicit DbnDecoder(detail::SharedChannel channel);
-  explicit DbnDecoder(detail::FileStream file_stream);
-  explicit DbnDecoder(std::unique_ptr<IReadable> input);
-  DbnDecoder(std::unique_ptr<IReadable> input,
+  DbnDecoder(ILogReceiver* log_receiver, detail::SharedChannel channel);
+  DbnDecoder(ILogReceiver* log_receiver, InFileStream file_stream);
+  DbnDecoder(ILogReceiver* log_receiver, std::unique_ptr<IReadable> input);
+  DbnDecoder(ILogReceiver* log_receiver, std::unique_ptr<IReadable> input,
              VersionUpgradePolicy upgrade_policy);
 
-  // Decode metadata from the given buffer.
-  static Metadata DecodeMetadata(const std::vector<std::uint8_t>& buffer);
   static std::pair<std::uint8_t, std::size_t> DecodeMetadataVersionAndSize(
       const std::uint8_t* buffer, std::size_t size);
   static Metadata DecodeMetadataFields(std::uint8_t version,
@@ -60,8 +59,10 @@ class DbnDecoder {
       std::vector<std::uint8_t>::const_iterator buffer_end_it);
   bool DetectCompression();
   std::size_t FillBuffer();
+  std::size_t GetReadBufferSize() const;
   RecordHeader* BufferRecordHeader();
 
+  ILogReceiver* log_receiver_;
   std::uint8_t version_{};
   VersionUpgradePolicy upgrade_policy_;
   bool ts_out_{};

include/databento/dbn_encoder.hpp

@@ -0,0 +1,39 @@
+#pragma once
+
+#include <cstdint>  // uint32_t
+
+#include "databento/dbn.hpp"  // Metadata
+#include "databento/iwritable.hpp"
+#include "databento/record.hpp"
+#include "databento/with_ts_out.hpp"
+
+namespace databento {
+class DbnEncoder {
+ public:
+  explicit DbnEncoder(const Metadata& metadata, IWritable* output);
+
+  static void EncodeMetadata(const Metadata& metadata, IWritable* output);
+  static void EncodeRecord(const Record& record, IWritable* output);
+
+  template <typename R>
+  void EncodeRecord(const R& record) {
+    static_assert(
+        has_header_v<R>,
+        "must be a DBN record struct with an `hd` RecordHeader field");
+    EncodeRecord(Record{&record.hd});
+  }
+  template <typename R>
+  void EncodeRecord(const WithTsOut<R> record) {
+    static_assert(
+        has_header_v<R>,
+        "must be a DBN record struct with an `hd` RecordHeader field");
+    EncodeRecord(Record{&record.rec.hd});
+  }
+  void EncodeRecord(const Record& record);
+
+ private:
+  static std::uint32_t CalcLength(const Metadata& metadata);
+
+  IWritable* output_;
+};
+}  // namespace databento