From f428100fdfb288145fd811655e4dd368568a960b Mon Sep 17 00:00:00 2001 From: Andrey Balabekyan <55596549+V0S7ER@users.noreply.github.com> Date: Sun, 2 Aug 2026 00:58:58 +0300 Subject: [PATCH 01/12] feat odbc: make driver production-ready --- .mapping.json | 12 +- odbc/README.md | 231 +------------- ...USERVER_ODBC_CONNECTION_POOL_SETTINGS.yaml | 5 +- .../USERVER_ODBC_DEFAULT_COMMAND_CONTROL.yaml | 12 +- odbc/functional_tests/CMakeLists.txt | 3 + .../basic_chaos/odbc_service.cpp | 36 ++- .../basic_chaos/static_config.yaml | 26 ++ .../basic_chaos/tests/test_metrics.py | 20 ++ .../secdist_update/CMakeLists.txt | 9 + .../secdist_update/odbc_service.cpp | 51 ++++ .../schemas/postgresql/key_value.sql | 1 + .../secdist_update/static_config.yaml | 57 ++++ .../secdist_update/tests/conftest.py | 45 +++ .../secdist_update/tests/test_secdist.py | 52 ++++ odbc/include/userver/storages/odbc.hpp | 1 + .../include/userver/storages/odbc/cluster.hpp | 45 ++- .../userver/storages/odbc/command_control.hpp | 29 ++ .../userver/storages/odbc/component.hpp | 35 ++- .../userver/storages/odbc/impl/parameter.hpp | 128 ++++++++ .../userver/storages/odbc/settings.hpp | 6 + .../userver/storages/odbc/transaction.hpp | 19 +- odbc/src/storages/odbc/cluster.cpp | 25 +- odbc/src/storages/odbc/component.cpp | 84 +++++- odbc/src/storages/odbc/component.yaml | 20 +- .../src/storages/odbc/detail/cluster_impl.cpp | 182 +++++++++-- .../src/storages/odbc/detail/cluster_impl.hpp | 50 +++- odbc/src/storages/odbc/detail/connection.cpp | 282 ++++++++++++++++-- odbc/src/storages/odbc/detail/connection.hpp | 7 + odbc/src/storages/odbc/detail/pool.cpp | 30 +- .../odbc/detail/topology/topology_base.cpp | 6 +- .../odbc/detail/topology/topology_base.hpp | 2 +- odbc/src/storages/odbc/odbc_config.hpp | 18 -- odbc/src/storages/odbc/odbc_secdist.cpp | 7 + odbc/src/storages/odbc/transaction.cpp | 22 +- odbc/tests/odbc_deadline_test.cpp | 26 +- odbc/tests/odbc_postgresql_test.cpp | 55 ++++ odbc/tests/odbc_transaction_test.cpp | 14 + scripts/docs/en/userver/odbc.md | 90 +++++- 38 files changed, 1353 insertions(+), 390 deletions(-) create mode 100644 odbc/functional_tests/basic_chaos/tests/test_metrics.py create mode 100644 odbc/functional_tests/secdist_update/CMakeLists.txt create mode 100644 odbc/functional_tests/secdist_update/odbc_service.cpp create mode 100644 odbc/functional_tests/secdist_update/schemas/postgresql/key_value.sql create mode 100644 odbc/functional_tests/secdist_update/static_config.yaml create mode 100644 odbc/functional_tests/secdist_update/tests/conftest.py create mode 100644 odbc/functional_tests/secdist_update/tests/test_secdist.py create mode 100644 odbc/include/userver/storages/odbc/command_control.hpp create mode 100644 odbc/include/userver/storages/odbc/impl/parameter.hpp delete mode 100644 odbc/src/storages/odbc/odbc_config.hpp diff --git a/.mapping.json b/.mapping.json index 684fa4ba249d..19dc4816636b 100644 --- a/.mapping.json +++ b/.mapping.json @@ -3640,14 +3640,23 @@ "odbc/functional_tests/basic_chaos/schemas/postgresql/key_value.sql":"taxi/uservices/userver/odbc/functional_tests/basic_chaos/schemas/postgresql/key_value.sql", "odbc/functional_tests/basic_chaos/static_config.yaml":"taxi/uservices/userver/odbc/functional_tests/basic_chaos/static_config.yaml", "odbc/functional_tests/basic_chaos/tests/conftest.py":"taxi/uservices/userver/odbc/functional_tests/basic_chaos/tests/conftest.py", + "odbc/functional_tests/basic_chaos/tests/test_metrics.py":"taxi/uservices/userver/odbc/functional_tests/basic_chaos/tests/test_metrics.py", "odbc/functional_tests/basic_chaos/tests/test_odbc.py":"taxi/uservices/userver/odbc/functional_tests/basic_chaos/tests/test_odbc.py", + "odbc/functional_tests/secdist_update/CMakeLists.txt":"taxi/uservices/userver/odbc/functional_tests/secdist_update/CMakeLists.txt", + "odbc/functional_tests/secdist_update/odbc_service.cpp":"taxi/uservices/userver/odbc/functional_tests/secdist_update/odbc_service.cpp", + "odbc/functional_tests/secdist_update/schemas/postgresql/key_value.sql":"taxi/uservices/userver/odbc/functional_tests/secdist_update/schemas/postgresql/key_value.sql", + "odbc/functional_tests/secdist_update/static_config.yaml":"taxi/uservices/userver/odbc/functional_tests/secdist_update/static_config.yaml", + "odbc/functional_tests/secdist_update/tests/conftest.py":"taxi/uservices/userver/odbc/functional_tests/secdist_update/tests/conftest.py", + "odbc/functional_tests/secdist_update/tests/test_secdist.py":"taxi/uservices/userver/odbc/functional_tests/secdist_update/tests/test_secdist.py", "odbc/include/userver/storages/odbc.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc.hpp", "odbc/include/userver/storages/odbc/cluster.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/cluster.hpp", "odbc/include/userver/storages/odbc/cluster_types.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/cluster_types.hpp", + "odbc/include/userver/storages/odbc/command_control.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/command_control.hpp", "odbc/include/userver/storages/odbc/component.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/component.hpp", "odbc/include/userver/storages/odbc/exception.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/exception.hpp", "odbc/include/userver/storages/odbc/execution_result.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/execution_result.hpp", "odbc/include/userver/storages/odbc/field.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/field.hpp", + "odbc/include/userver/storages/odbc/impl/parameter.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/impl/parameter.hpp", "odbc/include/userver/storages/odbc/impl/tracing_tags.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/impl/tracing_tags.hpp", "odbc/include/userver/storages/odbc/odbc_fwd.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/odbc_fwd.hpp", "odbc/include/userver/storages/odbc/query.hpp":"taxi/uservices/userver/odbc/include/userver/storages/odbc/query.hpp", @@ -3691,7 +3700,6 @@ "odbc/src/storages/odbc/dsn.hpp":"taxi/uservices/userver/odbc/src/storages/odbc/dsn.hpp", "odbc/src/storages/odbc/exception.cpp":"taxi/uservices/userver/odbc/src/storages/odbc/exception.cpp", "odbc/src/storages/odbc/field.cpp":"taxi/uservices/userver/odbc/src/storages/odbc/field.cpp", - "odbc/src/storages/odbc/odbc_config.hpp":"taxi/uservices/userver/odbc/src/storages/odbc/odbc_config.hpp", "odbc/src/storages/odbc/odbc_secdist.cpp":"taxi/uservices/userver/odbc/src/storages/odbc/odbc_secdist.cpp", "odbc/src/storages/odbc/odbc_secdist.hpp":"taxi/uservices/userver/odbc/src/storages/odbc/odbc_secdist.hpp", "odbc/src/storages/odbc/result_set.cpp":"taxi/uservices/userver/odbc/src/storages/odbc/result_set.cpp", @@ -6670,4 +6678,4 @@ "ydb/utest/include/userver/ydb/tests/topic_writer_mock.hpp":"taxi/uservices/userver/ydb/utest/include/userver/ydb/tests/topic_writer_mock.hpp", "ydb/utest/include/userver/ydb/tests/write_session_mock.hpp":"taxi/uservices/userver/ydb/utest/include/userver/ydb/tests/write_session_mock.hpp", "ydb/utest/src/utest/topic_writer_mock.cpp":"taxi/uservices/userver/ydb/utest/src/utest/topic_writer_mock.cpp" -} \ No newline at end of file +} diff --git a/odbc/README.md b/odbc/README.md index e1f9e26ac6da..689babcf0114 100644 --- a/odbc/README.md +++ b/odbc/README.md @@ -1,229 +1,4 @@ -# userver: ODBC Driver Wrapper [WIP] - -ODBC storage wrapper for `userver` (cluster + connection pool + query execution). - -Under active development! - -## Quick start - -Create a `storages::odbc::Cluster` with ODBC DSN and execute a query: - -```cpp -#include - -using namespace std::chrono_literals; - -storages::odbc::settings::PoolSettings pool_settings{ - .min_size=1, - .max_size=5, -}; - -storages::odbc::settings::HostSettings host_settings{ - .dsn="DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=15433;DATABASE=postgres;UID=testsuite;PWD=password;", - .pool=pool_settings, -}; - -storages::odbc::settings::ODBCClusterSettings cluster_settings{ - .pools={host_settings}, -}; - -storages::odbc::Cluster cluster{cluster_settings}; - -auto rs = cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELECT 1"); -auto row = rs[0]; -auto field = row[0]; -// field.GetInt32() / GetInt64() / GetString() / ... -``` - -### Reading results - -`Execute(...)` returns `storages::odbc::ResultSet`. Each row is `storages::odbc::Row`, and each field is `storages::odbc::Field`: - -```cpp -auto rs = cluster.Execute(storages::odbc::ClusterHostType::kMaster, - "SELECT 42, 'test', 1.0, false, null, true"); - -const auto row = rs[0]; -const auto i32 = row[0].GetInt32(); -const auto str = row[1].GetString(); -if (row[4].IsNull()) { - // ... -} -``` - -## Deadlines - -ODBC operations can be aborted when a deadline is reached. - -### Explicit deadline - -Use the overloads that accept `engine::Deadline`: - -```cpp -#include -#include - -using namespace std::chrono_literals; - -auto deadline = engine::Deadline::FromDuration(200ms); -auto rs = cluster.Execute(deadline, storages::odbc::ClusterHostType::kMaster, "SELECT 1"); -``` - -Deadlines are also applied to transactions started with `Begin(deadline, ...)`: - -```cpp -auto tx = cluster.Begin(deadline, storages::odbc::ClusterHostType::kMaster); -auto rs = tx.Execute("SELECT 1"); -tx.Commit(); // deadline is honored internally -``` - -### Deadline resolution in ODBC - -ODBC driver statement timeout (`SQL_ATTR_QUERY_TIMEOUT`) is configured in whole seconds. -As a result, when converting `engine::Deadline` to the ODBC timeout, sub-second deadlines are rounded up to the next full second (so the operation may run slightly longer than the exact deadline). - -### Request deadline propagation - -If you call ODBC from a request task, the task-inherited request deadline is automatically merged into ODBC deadlines. -If it expires, `storages::odbc::OperationInterrupted` is thrown. - -## Transactions - -Transactions are created via `Cluster::Begin(...)`. -They execute statements via `Transaction::Execute(...)`, then finish with `Commit()` or `Rollback()`. -If neither commit nor rollback was called, the transaction rolls back on destruction (RAII). - -```cpp -auto tx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); - -tx.Execute("INSERT INTO t(a) VALUES (1)"); -tx.Execute("UPDATE t SET a = a + 1 WHERE a = 1"); - -tx.Commit(); -``` - -## Exceptions - -Common exceptions from `userver::storages::odbc`: - -- `storages::odbc::OperationInterrupted` — deadline expired -- `storages::odbc::ConnectionError` — connection / driver failures -- `storages::odbc::StatementError` — statement-level execution errors - -## Component Configuration - -The ODBC component can be configured in the static config file. Below is the full schema: - -### Single Pool Configuration - -```yaml -components_manager: - components: - odbc: - dsn: "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;DATABASE=mydb;UID=user;PWD=password" - min_pool_size: 1 # optional, default: 1 - max_pool_size: 10 # optional, default: 10 - dns_resolver: async # optional, default: async (options: async, getaddrinfo) -``` - -### Multi-Pool Configuration - -For master-replica setups or multiple database hosts: - -```yaml -components_manager: - components: - odbc: - dns_resolver: async - pools: - - dsn: "DRIVER={PostgreSQL Unicode};SERVER=master.db.local;PORT=5432;DATABASE=mydb;UID=user;PWD=password" - min_pool_size: 2 - max_pool_size: 15 - - dsn: "DRIVER={PostgreSQL Unicode};SERVER=replica.db.local;PORT=5432;DATABASE=mydb;UID=user;PWD=password" - min_pool_size: 1 - max_pool_size: 10 -``` - -### Configuration Options - -| Option | Type | Default | Description | -|--------|------|---------|-------------| -| `secdist_alias` | string | — | Name of the database in secdist config (for secure credential storage) | -| `dsn` | string | — | ODBC connection string (for single-pool mode) | -| `min_pool_size` | integer | 1 | Minimum number of connections kept in the pool | -| `max_pool_size` | integer | 10 | Maximum number of connections in the pool | -| `dns_resolver` | string | `async` | DNS resolution mode: `async` (non-blocking) or `getaddrinfo` (blocking) | -| `pools` | array | — | List of pool configurations (for multi-pool mode) | - -### Secdist Integration - -For secure credential storage, you can use secdist instead of putting DSN strings directly in the static config: - -```yaml -components_manager: - components: - odbc: - secdist_alias: my_database - min_pool_size: 1 - max_pool_size: 10 - dns_resolver: async -``` - -The secdist JSON file should contain: - -```json -{ - "odbc_settings": { - "databases": { - "my_database": { - "dsn": "DRIVER={PostgreSQL Unicode};SERVER=localhost;PORT=5432;DATABASE=mydb;UID=user;PWD=secret" - } - } - } -} -``` - -For multiple hosts (master-replica setup): - -```json -{ - "odbc_settings": { - "databases": { - "my_database": { - "hosts": [ - "DRIVER={PostgreSQL Unicode};SERVER=master.db.local;PORT=5432;DATABASE=mydb;UID=user;PWD=secret", - "DRIVER={PostgreSQL Unicode};SERVER=replica.db.local;PORT=5432;DATABASE=mydb;UID=user;PWD=secret" - ] - } - } - } -} -``` - -### DNS Resolution - -The `dns_resolver` option controls how hostnames in DSN strings are resolved: - -- **`async`** (default): Uses userver's asynchronous DNS resolver. Hostnames are resolved at component startup and replaced with IP addresses in the DSN. This is non-blocking and recommended for production. - -- **`getaddrinfo`**: Uses the system's blocking `getaddrinfo()` call. The DSN is passed to the ODBC driver as-is, and hostname resolution happens during connection establishment. - -When using `async` mode, the SERVER/HOST parameter in your DSN will be automatically resolved to an IP address before connecting. This allows for proper integration with service discovery and DNS-based load balancing. - - -### Programmatic Access - -You can access the current dynamic config values programmatically: - -```cpp -#include - -// Get current default timeouts from dynamic config -auto network_timeout = cluster->GetDefaultNetworkTimeout(); -auto statement_timeout = cluster->GetDefaultStatementTimeout(); - -if (network_timeout.has_value()) { - // Use the configured timeout -} -``` +# userver ODBC driver +The driver documentation is maintained in +[`scripts/docs/en/userver/odbc.md`](../scripts/docs/en/userver/odbc.md). diff --git a/odbc/dynamic_configs/USERVER_ODBC_CONNECTION_POOL_SETTINGS.yaml b/odbc/dynamic_configs/USERVER_ODBC_CONNECTION_POOL_SETTINGS.yaml index 075d26ff8e9f..e1f9b51cf836 100644 --- a/odbc/dynamic_configs/USERVER_ODBC_CONNECTION_POOL_SETTINGS.yaml +++ b/odbc/dynamic_configs/USERVER_ODBC_CONNECTION_POOL_SETTINGS.yaml @@ -6,8 +6,9 @@ description: | The latter configuration is applied for every non-matching ODBC component of the service. - Note: Pool size changes require component restart to take effect, - as the underlying connection pool does not support dynamic resizing. + Updates atomically replace pools for new operations. Queries and + transactions already in progress keep their old pools alive until they + finish. schema: type: object example: | diff --git a/odbc/dynamic_configs/USERVER_ODBC_DEFAULT_COMMAND_CONTROL.yaml b/odbc/dynamic_configs/USERVER_ODBC_DEFAULT_COMMAND_CONTROL.yaml index 5ddc6456589b..8db2cd4bb8a5 100644 --- a/odbc/dynamic_configs/USERVER_ODBC_DEFAULT_COMMAND_CONTROL.yaml +++ b/odbc/dynamic_configs/USERVER_ODBC_DEFAULT_COMMAND_CONTROL.yaml @@ -1,8 +1,9 @@ default: {} description: | Dynamic config that controls default network and statement timeouts for ODBC driver. - Overrides the built-in timeouts, but could be overridden by explicit - engine::Deadline passed to Execute/Begin methods. + Overrides the built-in timeouts. Individual operations can override these + values with storages::odbc::OptionalCommandControl. A task-inherited + request deadline always caps the resulting deadline. schema: type: object additionalProperties: false @@ -12,8 +13,11 @@ schema: minimum: 1 x-usrv-cpp-type: std::chrono::milliseconds description: | - Network timeout in milliseconds. Controls how long to wait for - network operations (connection establishment, data transfer). + Overall operation budget in milliseconds. It starts before + connection acquisition, configures the ODBC login timeout, and + caps statement and transaction deadlines. Blocking ODBC calls + are ultimately subject to timeout support and whole-second + resolution of the selected ODBC driver. statement_timeout_ms: type: integer minimum: 1 diff --git a/odbc/functional_tests/CMakeLists.txt b/odbc/functional_tests/CMakeLists.txt index 6d5e1f776a94..0a208dc02779 100644 --- a/odbc/functional_tests/CMakeLists.txt +++ b/odbc/functional_tests/CMakeLists.txt @@ -4,3 +4,6 @@ add_custom_target(${PROJECT_NAME}) add_subdirectory(basic_chaos) add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-basic-chaos) + +add_subdirectory(secdist_update) +add_dependencies(${PROJECT_NAME} ${PROJECT_NAME}-secdist-update) diff --git a/odbc/functional_tests/basic_chaos/odbc_service.cpp b/odbc/functional_tests/basic_chaos/odbc_service.cpp index 3887a6d23c2b..59acc4746db1 100644 --- a/odbc/functional_tests/basic_chaos/odbc_service.cpp +++ b/odbc/functional_tests/basic_chaos/odbc_service.cpp @@ -1,12 +1,16 @@ #include #include +#include #include #include #include +#include +#include #include #include #include +#include #include namespace chaos { @@ -43,10 +47,8 @@ class KeyValue final : public server::handlers::HttpHandlerBase { private: std::string GetValue(std::string_view key, const server::http::HttpRequest& request) const { - auto result = odbc_->Execute( - storages::odbc::ClusterHostType::kMaster, - fmt::format("SELECT value FROM kv WHERE key = '{}'", key) - ); + auto result = + odbc_->Execute(storages::odbc::ClusterHostType::kMaster, "SELECT value FROM kv WHERE key = ?", key); if (result.IsEmpty()) { request.SetResponseStatus(server::http::HttpStatus::kNotFound); @@ -64,12 +66,10 @@ class KeyValue final : public server::handlers::HttpHandlerBase { odbc_->Execute( storages::odbc::ClusterHostType::kMaster, - fmt::format( - "INSERT INTO kv(key, value) VALUES ('{}', '{}') " - "ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", - key, - value - ) + "INSERT INTO kv(key, value) VALUES (?, ?) " + "ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", + key, + value ); request.SetResponseStatus(server::http::HttpStatus::kCreated); @@ -77,7 +77,7 @@ class KeyValue final : public server::handlers::HttpHandlerBase { } std::string DeleteValue(std::string_view key) const { - odbc_->Execute(storages::odbc::ClusterHostType::kMaster, fmt::format("DELETE FROM kv WHERE key = '{}'", key)); + odbc_->Execute(storages::odbc::ClusterHostType::kMaster, "DELETE FROM kv WHERE key = ?", key); return {}; } @@ -118,7 +118,7 @@ class KeyValueTrx final : public server::handlers::HttpHandlerBase { private: std::string GetValue(std::string_view key, const server::http::HttpRequest& request) const { auto trx = odbc_->Begin(storages::odbc::ClusterHostType::kMaster); - auto result = trx.Execute(fmt::format("SELECT value FROM kv WHERE key = '{}'", key)); + auto result = trx.Execute("SELECT value FROM kv WHERE key = ?", key); trx.Commit(); if (result.IsEmpty()) { @@ -136,12 +136,12 @@ class KeyValueTrx final : public server::handlers::HttpHandlerBase { } auto trx = odbc_->Begin(storages::odbc::ClusterHostType::kMaster); - trx.Execute(fmt::format( - "INSERT INTO kv(key, value) VALUES ('{}', '{}') " + trx.Execute( + "INSERT INTO kv(key, value) VALUES (?, ?) " "ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value", key, value - )); + ); trx.Commit(); request.SetResponseStatus(server::http::HttpStatus::kCreated); @@ -150,7 +150,7 @@ class KeyValueTrx final : public server::handlers::HttpHandlerBase { std::string DeleteValue(std::string_view key) const { auto trx = odbc_->Begin(storages::odbc::ClusterHostType::kMaster); - trx.Execute(fmt::format("DELETE FROM kv WHERE key = '{}'", key)); + trx.Execute("DELETE FROM kv WHERE key = ?", key); trx.Commit(); return {}; @@ -164,8 +164,12 @@ class KeyValueTrx final : public server::handlers::HttpHandlerBase { int main(int argc, char* argv[]) { const auto component_list = components::MinimalServerComponentList() + .AppendComponentList(clients::http::ComponentList()) .Append() .Append() + .Append() + .Append() + .Append() .Append() .Append() .Append() diff --git a/odbc/functional_tests/basic_chaos/static_config.yaml b/odbc/functional_tests/basic_chaos/static_config.yaml index 73edd4461515..7644c252a792 100644 --- a/odbc/functional_tests/basic_chaos/static_config.yaml +++ b/odbc/functional_tests/basic_chaos/static_config.yaml @@ -10,10 +10,12 @@ components_manager: task_processor: main-task-processor method: GET,POST,DELETE + # [ODBC component config] key-value-db: secdist_alias: key-value-db min_pool_size: 1 max_pool_size: 1 + # [ODBC component config] secdist: {} default-secdist-provider: @@ -25,6 +27,30 @@ components_manager: listener: port: 8097 task_processor: main-task-processor + listener-monitor: + port: $monitor-server-port + port#fallback: 8098 + task_processor: main-task-processor + + handler-server-monitor: + path: /service/monitor + method: GET + task_processor: main-task-processor + + testsuite-support: + + http-client: {} + http-client-core: + fs-task-processor: fs-task-processor + + tests-control: + method: POST + path: /tests/{action} + task_processor: main-task-processor + testpoint-url: $mockserver/testpoint + testpoint-timeout: 10s + skip-unregistered-testpoints: true + throttling_enabled: false logging: fs-task-processor: fs-task-processor loggers: diff --git a/odbc/functional_tests/basic_chaos/tests/test_metrics.py b/odbc/functional_tests/basic_chaos/tests/test_metrics.py new file mode 100644 index 000000000000..4c9c360b69dd --- /dev/null +++ b/odbc/functional_tests/basic_chaos/tests/test_metrics.py @@ -0,0 +1,20 @@ +async def test_odbc_metrics_smoke(service_client, monitor_client): + response = await service_client.post('/chaos?key=metrics&value=value') + assert response.status == 201 + + response = await service_client.get('/chaos/trx?key=metrics') + assert response.status == 200 + + metrics = await monitor_client.metrics_raw(output_format='pretty') + odbc_metrics = [line for line in metrics.splitlines() if line.startswith('odbc.')] + + assert odbc_metrics + assert any('component=key-value-db' in line for line in odbc_metrics) + assert any('odbc_pool=0' in line for line in odbc_metrics) + assert any(line.startswith('odbc.queries.executed') for line in odbc_metrics) + assert any(line.startswith('odbc.transactions.committed') for line in odbc_metrics) + + +async def test_odbc_metrics_portability(service_client): + warnings = await service_client.metrics_portability() + assert not warnings diff --git a/odbc/functional_tests/secdist_update/CMakeLists.txt b/odbc/functional_tests/secdist_update/CMakeLists.txt new file mode 100644 index 000000000000..a16d090a637a --- /dev/null +++ b/odbc/functional_tests/secdist_update/CMakeLists.txt @@ -0,0 +1,9 @@ +project(userver-odbc-tests-secdist-update CXX) + +add_executable(${PROJECT_NAME} "odbc_service.cpp") +target_link_libraries(${PROJECT_NAME} userver::odbc) + +userver_chaos_testsuite_add( + ENV "TESTSUITE_PGSQL_SERVER_START_TIMEOUT=120.0" + RESOURCE_LOCKS userver_postgresql +) diff --git a/odbc/functional_tests/secdist_update/odbc_service.cpp b/odbc/functional_tests/secdist_update/odbc_service.cpp new file mode 100644 index 000000000000..dc7873e05690 --- /dev/null +++ b/odbc/functional_tests/secdist_update/odbc_service.cpp @@ -0,0 +1,51 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace odbc::secdist_update { + +class Handler final : public server::handlers::HttpHandlerBase { +public: + static constexpr std::string_view kName{"handler-odbc"}; + + Handler(const components::ComponentConfig& config, const components::ComponentContext& context) + : server::handlers::HttpHandlerBase{config, context}, + // Deliberately cache the stable Cluster identity. A secdist update + // must reconfigure this object in place. + cluster_{context.FindComponent("odbc-database").GetCluster()} + {} + + std::string HandleRequestThrow(const server::http::HttpRequest&, server::request::RequestContext&) const override { + const auto result = cluster_->Execute(storages::odbc::ClusterHostType::kMaster, "SELECT ?::integer", 42); + return std::to_string(result[0][0].GetInt32()); + } + +private: + const std::shared_ptr cluster_; +}; + +} // namespace odbc::secdist_update + +int main(int argc, char* argv[]) { + const auto component_list = + components::MinimalServerComponentList() + .AppendComponentList(clients::http::ComponentList()) + .Append() + .Append() + .Append() + .Append() + .Append() + .Append() + .Append("odbc-database"); + return utils::DaemonMain(argc, argv, component_list); +} diff --git a/odbc/functional_tests/secdist_update/schemas/postgresql/key_value.sql b/odbc/functional_tests/secdist_update/schemas/postgresql/key_value.sql new file mode 100644 index 000000000000..e0ac49d1ecfb --- /dev/null +++ b/odbc/functional_tests/secdist_update/schemas/postgresql/key_value.sql @@ -0,0 +1 @@ +SELECT 1; diff --git a/odbc/functional_tests/secdist_update/static_config.yaml b/odbc/functional_tests/secdist_update/static_config.yaml new file mode 100644 index 000000000000..0c5b91ed8b38 --- /dev/null +++ b/odbc/functional_tests/secdist_update/static_config.yaml @@ -0,0 +1,57 @@ +components_manager: + components: + handler-odbc: + path: /odbc + task_processor: main-task-processor + method: GET + + odbc-database: + secdist_alias: odbc-test + min_pool_size: 0 + max_pool_size: 1 + dns_resolver: getaddrinfo + + testsuite-support: + + http-client: {} + http-client-core: + fs-task-processor: fs-task-processor + + tests-control: + method: POST + path: /tests/{action} + task_processor: main-task-processor + testpoint-url: $mockserver/testpoint + testpoint-timeout: 10s + skip-unregistered-testpoints: true + throttling_enabled: false + + secdist: + load-enabled: true + update-period: 1s + default-secdist-provider: + config: /etc/odbc_service/secdist.json + + server: + listener: + port: 8099 + task_processor: main-task-processor + + logging: + fs-task-processor: fs-task-processor + loggers: + default: + file_path: '@stderr' + level: info + overflow_behavior: discard + + dns-client: + fs-task-processor: fs-task-processor + + task_processors: + main-task-processor: + worker_threads: 4 + fs-task-processor: + worker_threads: 2 + + default_task_processor: main-task-processor diff --git a/odbc/functional_tests/secdist_update/tests/conftest.py b/odbc/functional_tests/secdist_update/tests/conftest.py new file mode 100644 index 000000000000..e9fb9e4e6de5 --- /dev/null +++ b/odbc/functional_tests/secdist_update/tests/conftest.py @@ -0,0 +1,45 @@ +import json + +import pytest +from testsuite.databases.pgsql import discover + +pytest_plugins = ['pytest_userver.plugins.postgresql'] +USERVER_CONFIG_HOOKS = ['userver_config_secdist'] + + +@pytest.fixture(name='pgsql_local', scope='session') +def _pgsql_local(service_source_dir, pgsql_local_create): + databases = discover.find_schemas( + 'pg', + [service_source_dir.joinpath('schemas/postgresql')], + ) + return pgsql_local_create(list(databases.values())) + + +@pytest.fixture(scope='session') +def secdist_path(service_tmpdir): + path = service_tmpdir / 'secdist.json' + path.write_text( + json.dumps({ + 'odbc_settings': { + 'databases': { + 'odbc-test': { + 'dsn': ( + 'Driver={PostgreSQL Unicode};Server=localhost;' + 'Port=1;Database=postgres;Uid=testsuite;Pwd=;' + ), + }, + }, + }, + }), + ) + return path + + +@pytest.fixture(scope='session') +def userver_config_secdist(secdist_path): + def _hook(config_yaml, _config_vars): + components = config_yaml['components_manager']['components'] + components['default-secdist-provider']['config'] = str(secdist_path) + + return _hook diff --git a/odbc/functional_tests/secdist_update/tests/test_secdist.py b/odbc/functional_tests/secdist_update/tests/test_secdist.py new file mode 100644 index 000000000000..21c23353e5e9 --- /dev/null +++ b/odbc/functional_tests/secdist_update/tests/test_secdist.py @@ -0,0 +1,52 @@ +import asyncio +import json +import os + + +def _valid_dsn(pgsql_local): + database = pgsql_local['key_value'] + return ( + f'Driver={{PostgreSQL Unicode}};Server={database.host};' + f'Port={database.port};Database={database.dbname};' + f'Uid={database.user or "testsuite"};Pwd={database.password or ""};' + ) + + +def _replace_secdist(path, dsn): + temporary = path.with_suffix('.tmp') + temporary.write_text( + json.dumps({ + 'odbc_settings': { + 'databases': {'odbc-test': {'dsn': dsn}}, + }, + }), + ) + os.replace(temporary, path) + + +async def test_secdist_hot_reload(service_client, secdist_path, pgsql_local, testpoint): + failed = await service_client.get('/odbc') + assert failed.status == 500 + + @testpoint('odbc-new-dsn-list') + def new_dsn_list(_data): + pass + + await service_client.update_server_state() + + # Periodic secdist notifications with unchanged data must not rebuild all + # pools or reset their metrics. + await asyncio.sleep(1.2) + assert new_dsn_list.times_called == 0 + + _replace_secdist(secdist_path, _valid_dsn(pgsql_local)) + await new_dsn_list.wait_call(timeout=10) + + for _ in range(20): + response = await service_client.get('/odbc') + if response.status == 200: + assert response.text == '42' + return + await asyncio.sleep(0.1) + + raise AssertionError('cached ODBC cluster did not recover after secdist update') diff --git a/odbc/include/userver/storages/odbc.hpp b/odbc/include/userver/storages/odbc.hpp index 5857f22d88d9..898db6f960e6 100644 --- a/odbc/include/userver/storages/odbc.hpp +++ b/odbc/include/userver/storages/odbc.hpp @@ -5,6 +5,7 @@ /// that are required for working with ODBC µserver component. #include +#include #include #include diff --git a/odbc/include/userver/storages/odbc/cluster.hpp b/odbc/include/userver/storages/odbc/cluster.hpp index 92c87ceb6d45..e88403d88df5 100644 --- a/odbc/include/userver/storages/odbc/cluster.hpp +++ b/odbc/include/userver/storages/odbc/cluster.hpp @@ -7,10 +7,11 @@ #include #include -#include #include #include +#include +#include #include #include #include @@ -20,8 +21,6 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc { -struct CommandControl; - namespace detail { class ClusterImpl; @@ -36,19 +35,44 @@ class Cluster { ~Cluster(); - ResultSet Execute(ClusterHostTypeFlags flags, const Query& query); - - ResultSet Execute(engine::Deadline deadline, ClusterHostTypeFlags flags, const Query& query); + /// @brief Execute a statement, binding every argument to an ODBC `?` placeholder. + /// + /// @warning Never interpolate untrusted values into @p query. Passing them as + /// separate arguments ensures that they are sent to the ODBC driver as data. + template + ResultSet Execute(ClusterHostTypeFlags flags, const Query& query, const Args&... args) { + return Execute(flags, std::nullopt, query, args...); + } + + /// @brief Execute a statement with per-operation timeout overrides. + template + ResultSet Execute( + ClusterHostTypeFlags flags, + OptionalCommandControl command_control, + const Query& query, + const Args&... args + ) { + return DoExecute(command_control, flags, query, impl::MakeParameterList(args...)); + } Transaction Begin(ClusterHostTypeFlags flags); - Transaction Begin(engine::Deadline deadline, ClusterHostTypeFlags flags); + Transaction Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control); void WriteStatistics(utils::statistics::Writer& writer) const; /// @brief Set default command control (timeouts) from dynamic config void SetDefaultCommandControl(const CommandControl& cc); + /// @brief Atomically replace cluster pools for future operations. + /// Existing queries and transactions keep their old pools alive. + void UpdateSettings(const settings::ODBCClusterSettings& settings); + + /// @cond + void UpdateDsns(const std::vector& dsns); + void SetPoolSettingsOverride(std::optional settings); + /// @endcond + /// @brief Get current default network timeout std::optional GetDefaultNetworkTimeout() const; @@ -56,6 +80,13 @@ class Cluster { std::optional GetDefaultStatementTimeout() const; private: + ResultSet DoExecute( + OptionalCommandControl command_control, + ClusterHostTypeFlags flags, + const Query& query, + const impl::ParameterList& parameters + ); + detail::ClusterImplPtr impl_; }; diff --git a/odbc/include/userver/storages/odbc/command_control.hpp b/odbc/include/userver/storages/odbc/command_control.hpp new file mode 100644 index 000000000000..b6201dd5f90f --- /dev/null +++ b/odbc/include/userver/storages/odbc/command_control.hpp @@ -0,0 +1,29 @@ +#pragma once + +/// @file userver/storages/odbc/command_control.hpp +/// @brief Per-operation timeout settings for the ODBC driver. + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc { + +struct CommandControl final { + /// Overall operation budget used for pool waiting and connection login and + /// as an upper bound for statement/transaction deadlines. Blocking ODBC + /// calls are ultimately subject to the timeout capabilities and whole- + /// second resolution of the selected ODBC driver. + std::optional network_timeout; + + /// Timeout for statement execution. ODBC drivers accept this timeout in + /// whole seconds, so the value is rounded up when passed to a driver. + std::optional statement_timeout; +}; + +using OptionalCommandControl = std::optional; + +} // namespace storages::odbc + +USERVER_NAMESPACE_END diff --git a/odbc/include/userver/storages/odbc/component.hpp b/odbc/include/userver/storages/odbc/component.hpp index f86147a4508c..b3ef012aaf7c 100644 --- a/odbc/include/userver/storages/odbc/component.hpp +++ b/odbc/include/userver/storages/odbc/component.hpp @@ -4,9 +4,14 @@ /// @brief @copybrief components::Odbc #include +#include +#include +#include #include #include +#include +#include USERVER_NAMESPACE_BEGIN @@ -16,7 +21,27 @@ class Cluster; namespace components { -/// @brief Component that owns a storages::odbc::Cluster +/// @ingroup userver_components +/// +/// @brief ODBC client component that owns a storages::odbc::Cluster. +/// +/// ## Dynamic options: +/// * @ref USERVER_ODBC_DEFAULT_COMMAND_CONTROL +/// * @ref USERVER_ODBC_CONNECTION_POOL_SETTINGS +/// +/// ## Static configuration example: +/// +/// @snippet odbc/functional_tests/basic_chaos/static_config.yaml ODBC component config +/// +/// Exactly one of `dsn`, `pools`, and `secdist_alias` must be specified. +/// With `secdist_alias`, connection data is loaded from components::Secdist and +/// is updated without changing the Cluster object returned by GetCluster(). +/// +/// ## Static options of components::Odbc: +/// @include{doc} scripts/docs/en/components_schema/odbc/src/storages/odbc/component.md +/// +/// Options inherited from @ref components::ComponentBase: +/// @include{doc} scripts/docs/en/components_schema/core/src/components/impl/component_base.md class Odbc final : public ComponentBase { public: static constexpr std::string_view kName = "odbc"; @@ -30,14 +55,22 @@ class Odbc final : public ComponentBase { private: void OnConfigUpdate(const dynamic_config::Snapshot& config); + void OnSecdistUpdate(const storages::secdist::SecdistConfig& secdist); std::string name_; + std::optional secdist_alias_; std::shared_ptr cluster_; dynamic_config::Source config_source_; + + // Subscriptions must be the last fields because callbacks use all fields above. concurrent::AsyncEventSubscriberScope config_subscription_; + concurrent::AsyncEventSubscriberScope secdist_subscription_; }; +template <> +inline constexpr bool kHasValidate = true; + } // namespace components USERVER_NAMESPACE_END diff --git a/odbc/include/userver/storages/odbc/impl/parameter.hpp b/odbc/include/userver/storages/odbc/impl/parameter.hpp new file mode 100644 index 000000000000..019b620269c4 --- /dev/null +++ b/odbc/include/userver/storages/odbc/impl/parameter.hpp @@ -0,0 +1,128 @@ +#pragma once + +/// @file userver/storages/odbc/impl/parameter.hpp +/// @brief Internal storage for ODBC query parameters. + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc::impl { + +enum class ParameterType { + kBoolean, + kSignedInteger, + kUnsignedInteger, + kFloatingPoint, + kString, + kUnknown, +}; + +/// A type-erased, owning query parameter. Owning the value is important because +/// an ODBC driver is allowed to read bound buffers until SQLExecute returns. +class Parameter final { +public: + using Value = std::variant; + + Parameter(std::nullptr_t) + : type_{ParameterType::kUnknown}, + is_null_{true}, + value_{std::string{}} + {} + Parameter(std::nullopt_t) + : Parameter{nullptr} + {} + + Parameter(bool value) + : type_{ParameterType::kBoolean}, + value_{value} + {} + + template + requires(!std::same_as) + Parameter(T value) + : type_{ParameterType::kSignedInteger}, + value_{static_cast(value)} + {} + + template + requires(!std::same_as) + Parameter(T value) + : type_{ParameterType::kUnsignedInteger}, + value_{static_cast(value)} + {} + + template + Parameter(T value) + : type_{ParameterType::kFloatingPoint}, + value_{static_cast(value)} + {} + + template + requires std::is_enum_v + Parameter(T value) + : Parameter{static_cast>(value)} + {} + + Parameter(const char* value) + : type_{ParameterType::kString}, + is_null_{value == nullptr}, + value_{value == nullptr ? std::string{} : std::string{value}} + {} + Parameter(std::string value) + : type_{ParameterType::kString}, + value_{std::move(value)} + {} + Parameter(std::string_view value) + : Parameter{std::string{value}} + {} + + template + Parameter(const std::optional& value) + : Parameter{value ? Parameter{*value} : NullOf()} + {} + + ParameterType GetType() const noexcept { return type_; } + bool IsNull() const noexcept { return is_null_; } + + template + const T& Get() const { + return std::get(value_); + } + +private: + template + static Parameter NullOf() { + Parameter result{T{}}; + result.is_null_ = true; + return result; + } + + ParameterType type_; + bool is_null_{false}; + Value value_; +}; + +using ParameterList = std::vector; + +template +ParameterList MakeParameterList(const Args&... args) { + ParameterList result; + result.reserve(sizeof...(Args)); + (result.emplace_back(args), ...); + return result; +} + +} // namespace storages::odbc::impl + +USERVER_NAMESPACE_END diff --git a/odbc/include/userver/storages/odbc/settings.hpp b/odbc/include/userver/storages/odbc/settings.hpp index e408f6b830c9..65715fb6a396 100644 --- a/odbc/include/userver/storages/odbc/settings.hpp +++ b/odbc/include/userver/storages/odbc/settings.hpp @@ -13,15 +13,21 @@ namespace storages::odbc::settings { struct PoolSettings final { std::size_t min_size{5}; std::size_t max_size{10}; + + bool operator==(const PoolSettings&) const = default; }; struct HostSettings final { const std::string dsn; PoolSettings pool; + + bool operator==(const HostSettings&) const = default; }; struct ODBCClusterSettings final { std::vector pools; + + bool operator==(const ODBCClusterSettings&) const = default; }; } // namespace storages::odbc::settings diff --git a/odbc/include/userver/storages/odbc/transaction.hpp b/odbc/include/userver/storages/odbc/transaction.hpp index 4d7eec58bf4c..002c0b63ff12 100644 --- a/odbc/include/userver/storages/odbc/transaction.hpp +++ b/odbc/include/userver/storages/odbc/transaction.hpp @@ -10,6 +10,8 @@ #include #include +#include +#include #include #include @@ -34,7 +36,17 @@ class Transaction final { Transaction(const Transaction& other) = delete; Transaction(Transaction&& other) noexcept; - ResultSet Execute(const Query& query); + /// @brief Execute a statement, binding every argument to an ODBC `?` placeholder. + template + ResultSet Execute(const Query& query, const Args&... args) { + return Execute(std::nullopt, query, args...); + } + + /// @brief Execute a statement with per-statement timeout overrides. + template + ResultSet Execute(OptionalCommandControl command_control, const Query& query, const Args&... args) { + return DoExecute(command_control, query, impl::MakeParameterList(args...)); + } /// @brief Commit the transaction void Commit(); @@ -43,6 +55,11 @@ class Transaction final { void Rollback(); private: + ResultSet DoExecute( + OptionalCommandControl command_control, + const Query& query, + const impl::ParameterList& parameters + ); void AssertValid() const; // shared_ptr(16) + unique_ptr(8) = 24 bytes, align 8 diff --git a/odbc/src/storages/odbc/cluster.cpp b/odbc/src/storages/odbc/cluster.cpp index cf39a92042e5..1d9a79c40469 100644 --- a/odbc/src/storages/odbc/cluster.cpp +++ b/odbc/src/storages/odbc/cluster.cpp @@ -1,7 +1,7 @@ #include #include -#include +#include #include @@ -17,22 +17,33 @@ Cluster::Cluster(const settings::ODBCClusterSettings& settings, clients::dns::Re Cluster::~Cluster() = default; -ResultSet Cluster::Execute(ClusterHostTypeFlags flags, const Query& query) { return impl_->Execute(flags, query); } - -ResultSet Cluster::Execute(engine::Deadline deadline, ClusterHostTypeFlags flags, const Query& query) { - return impl_->Execute(deadline, flags, query); +ResultSet Cluster::DoExecute( + OptionalCommandControl command_control, + ClusterHostTypeFlags flags, + const Query& query, + const impl::ParameterList& parameters +) { + return impl_->Execute(flags, command_control, query, parameters); } Transaction Cluster::Begin(ClusterHostTypeFlags flags) { return impl_->Begin(flags); } -Transaction Cluster::Begin(engine::Deadline deadline, ClusterHostTypeFlags flags) { - return impl_->Begin(deadline, flags); +Transaction Cluster::Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control) { + return impl_->Begin(flags, command_control); } void Cluster::WriteStatistics(utils::statistics::Writer& writer) const { impl_->WriteStatistics(writer); } void Cluster::SetDefaultCommandControl(const CommandControl& cc) { impl_->SetDefaultCommandControl(cc); } +void Cluster::UpdateSettings(const settings::ODBCClusterSettings& settings) { impl_->UpdateSettings(settings); } + +void Cluster::UpdateDsns(const std::vector& dsns) { impl_->UpdateDsns(dsns); } + +void Cluster::SetPoolSettingsOverride(std::optional settings) { + impl_->SetPoolSettingsOverride(settings); +} + std::optional Cluster::GetDefaultNetworkTimeout() const { return impl_->GetDefaultNetworkTimeout(); } diff --git a/odbc/src/storages/odbc/component.cpp b/odbc/src/storages/odbc/component.cpp index 2dff078310d9..ccb8535e743c 100644 --- a/odbc/src/storages/odbc/component.cpp +++ b/odbc/src/storages/odbc/component.cpp @@ -1,6 +1,9 @@ #include #include +#include +#include +#include #include #include @@ -16,9 +19,10 @@ #include #include -#include "odbc_config.hpp" #include "odbc_secdist.hpp" +#include + #include #include @@ -32,16 +36,29 @@ namespace components { namespace { +void ValidatePoolSettings(const storages::odbc::settings::PoolSettings& settings) { + UINVARIANT(settings.max_size > 0, "ODBC max_pool_size must be positive"); + UINVARIANT(settings.min_size <= settings.max_size, "ODBC min_pool_size must not exceed max_pool_size"); +} + +void ValidateNonEmpty(std::string_view value, std::string_view option) { + if (value.empty()) { + throw std::runtime_error("ODBC component option '" + std::string{option} + "' must not be empty"); + } +} + storages::odbc::settings::ODBCClusterSettings MakeClusterSettingsFromConfig(const components::ComponentConfig& config) { using storages::odbc::settings::HostSettings; using storages::odbc::settings::ODBCClusterSettings; using storages::odbc::settings::PoolSettings; if (const auto dsn_opt = config["dsn"].As>(); dsn_opt.has_value()) { + ValidateNonEmpty(*dsn_opt, "dsn"); const auto min_size = config["min_pool_size"].As(PoolSettings{}.min_size); const auto max_size = config["max_pool_size"].As(PoolSettings{}.max_size); + ValidatePoolSettings(PoolSettings{.min_size = min_size, .max_size = max_size}); return ODBCClusterSettings{std::vector{ - HostSettings{*dsn_opt, PoolSettings{min_size, max_size}}, + HostSettings{.dsn = *dsn_opt, .pool = {.min_size = min_size, .max_size = max_size}}, }}; } @@ -52,9 +69,14 @@ storages::odbc::settings::ODBCClusterSettings MakeClusterSettingsFromConfig(cons for (std::size_t i = 0; i < pools_cfg.GetSize(); ++i) { const auto pool = pools_cfg[i]; const auto dsn = pool["dsn"].As(); + ValidateNonEmpty(dsn, "pools[" + std::to_string(i) + "].dsn"); const auto min_size = pool["min_pool_size"].As(PoolSettings{}.min_size); const auto max_size = pool["max_pool_size"].As(PoolSettings{}.max_size); - pools.emplace_back(HostSettings{dsn, PoolSettings{min_size, max_size}}); + ValidatePoolSettings(PoolSettings{.min_size = min_size, .max_size = max_size}); + pools.emplace_back(HostSettings{ + .dsn = dsn, + .pool = {.min_size = min_size, .max_size = max_size}, + }); } return ODBCClusterSettings{std::move(pools)}; } @@ -75,11 +97,15 @@ storages::odbc::settings::ODBCClusterSettings MakeClusterSettingsFromSecdist( const auto min_size = config["min_pool_size"].As(PoolSettings{}.min_size); const auto max_size = config["max_pool_size"].As(PoolSettings{}.max_size); + ValidatePoolSettings(PoolSettings{.min_size = min_size, .max_size = max_size}); std::vector pools; pools.reserve(connection_infos.size()); for (const auto& info : connection_infos) { - pools.emplace_back(HostSettings{info.dsn, PoolSettings{min_size, max_size}}); + pools.emplace_back(HostSettings{ + .dsn = info.dsn, + .pool = {.min_size = min_size, .max_size = max_size}, + }); } return ODBCClusterSettings{std::move(pools)}; @@ -90,16 +116,26 @@ storages::odbc::settings::ODBCClusterSettings MakeClusterSettings( const components::ComponentContext& context ) { const auto secdist_alias = config["secdist_alias"].As>(); + const auto dsn = config["dsn"].As>(); + const auto pools = config["pools"]; + const auto has_pools = !pools.IsMissing() && pools.GetSize() > 0; + + const auto connection_sources = + static_cast(secdist_alias.has_value()) + static_cast(dsn.has_value()) + + static_cast(has_pools); + UINVARIANT( + connection_sources == 1, + "Exactly one ODBC connection source must be configured: 'dsn', 'pools', or 'secdist_alias'" + ); if (secdist_alias.has_value()) { + ValidateNonEmpty(*secdist_alias, "secdist_alias"); const auto& secdist = context.FindComponent(); const auto& odbc_settings = secdist.Get().Get(); return MakeClusterSettingsFromSecdist(odbc_settings, *secdist_alias, config); } - auto settings = MakeClusterSettingsFromConfig(config); - UINVARIANT(!settings.pools.empty(), "Either 'dsn', 'pools', or 'secdist_alias' must be set"); - return settings; + return MakeClusterSettingsFromConfig(config); } } // namespace @@ -107,6 +143,7 @@ storages::odbc::settings::ODBCClusterSettings MakeClusterSettings( Odbc::Odbc(const ComponentConfig& config, const ComponentContext& context) : ComponentBase{config, context}, name_{config.Name()}, + secdist_alias_{config["secdist_alias"].As>()}, cluster_{std::make_shared< storages::odbc::Cluster>(MakeClusterSettings(config, context), clients::dns::GetResolverPtr(config, context)) }, @@ -127,20 +164,32 @@ Odbc::Odbc(const ComponentConfig& config, const ComponentContext& context) ::dynamic_config::USERVER_ODBC_CONNECTION_POOL_SETTINGS, ::dynamic_config::USERVER_ODBC_DEFAULT_COMMAND_CONTROL ); + + if (secdist_alias_) { + auto& secdist = context.FindComponent(); + secdist_subscription_ = secdist.GetStorage().UpdateAndListen(this, name_, &Odbc::OnSecdistUpdate); + } } -Odbc::~Odbc() { config_subscription_.Unsubscribe(); } +Odbc::~Odbc() { + config_subscription_.Unsubscribe(); + secdist_subscription_.Unsubscribe(); +} void Odbc::OnConfigUpdate(const dynamic_config::Snapshot& config) { const auto& pool_settings = config[::dynamic_config::USERVER_ODBC_CONNECTION_POOL_SETTINGS]; // Apply default command control from dynamic config const auto pool_settings_opt = pool_settings.GetOptional(name_); + std::optional updated; if (pool_settings_opt.has_value()) { - // Note: Pool size changes require restart as ConnectionPoolBase - // doesn't support dynamic resizing. Log a warning if settings differ. - // In future versions, this could be enhanced to support dynamic resizing. + updated = storages::odbc::settings::PoolSettings{ + .min_size = pool_settings_opt->min_pool_size, + .max_size = pool_settings_opt->max_pool_size, + }; + ValidatePoolSettings(*updated); } + cluster_->SetPoolSettingsOverride(updated); // Apply command control (timeouts) const auto cc = config[::dynamic_config::USERVER_ODBC_DEFAULT_COMMAND_CONTROL]; @@ -150,6 +199,19 @@ void Odbc::OnConfigUpdate(const dynamic_config::Snapshot& config) { }); } +void Odbc::OnSecdistUpdate(const storages::secdist::SecdistConfig& secdist) { + UASSERT(secdist_alias_); + const auto& odbc_settings = secdist.Get(); + const auto connection_infos = odbc_settings.GetConnectionInfos(*secdist_alias_); + + std::vector dsns; + dsns.reserve(connection_infos.size()); + for (const auto& info : connection_infos) { + dsns.push_back(info.dsn); + } + cluster_->UpdateDsns(dsns); +} + std::shared_ptr Odbc::GetCluster() const { return cluster_; } yaml_config::Schema Odbc::GetStaticConfigSchema() { diff --git a/odbc/src/storages/odbc/component.yaml b/odbc/src/storages/odbc/component.yaml index 4aa03947941a..066bcb7d2318 100644 --- a/odbc/src/storages/odbc/component.yaml +++ b/odbc/src/storages/odbc/component.yaml @@ -6,18 +6,25 @@ properties: type: string description: | name of the database in secdist config. + Must not be empty. If specified, DSN will be read from secdist instead of static config. + Mutually exclusive with `dsn` and `pools`. dsn: type: string - description: connection DSN string (used for single-pool configuration) + description: | + connection DSN string (used for single-pool configuration). + Must not be empty. + Mutually exclusive with `secdist_alias` and `pools`. min_pool_size: type: integer + minimum: 0 description: | number of connections created initially by this component instance. Connections are kept even without requests default: 1 max_pool_size: type: integer + minimum: 1 description: | maximum number of connections that can be created by this component instance. Should not be less than `min_pool_size` @@ -31,7 +38,10 @@ properties: - async pools: type: array - description: list of connection pools (used for multi-pool configuration) + minItems: 1 + description: | + list of connection pools (used for multi-pool configuration). + Mutually exclusive with `secdist_alias` and `dsn`. items: type: object description: connection pool configuration @@ -39,12 +49,16 @@ properties: properties: dsn: type: string - description: connection DSN string for this pool + description: non-empty connection DSN string for this pool min_pool_size: type: integer + minimum: 0 description: minimum number of connections in this pool default: 1 max_pool_size: type: integer + minimum: 1 description: maximum number of connections in this pool default: 10 + required: + - dsn diff --git a/odbc/src/storages/odbc/detail/cluster_impl.cpp b/odbc/src/storages/odbc/detail/cluster_impl.cpp index 656962cc61f1..c7a29cf8d237 100644 --- a/odbc/src/storages/odbc/detail/cluster_impl.cpp +++ b/odbc/src/storages/odbc/detail/cluster_impl.cpp @@ -1,13 +1,17 @@ #include +#include +#include + #include #include #include -#include #include +#include #include #include +#include #include #include #include @@ -18,35 +22,50 @@ namespace storages::odbc::detail { static_assert(std::atomic::is_always_lock_free); -ClusterImpl::ClusterImpl(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) { +ClusterImpl::ClusterImpl(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) + : resolver_{resolver}, + settings_{std::make_shared(settings)}, + baseline_settings_{settings_} +{ UINVARIANT(!settings.pools.empty(), "Pools count should be positive"); - topology_ = topology::TopologyBase::Create(settings, resolver); -} - -ResultSet ClusterImpl::Execute([[maybe_unused]] ClusterHostTypeFlags flags, const Query& query) { - return ExecuteImpl(GetExecuteDeadline(kDefaultStatementTimeout), flags, query); + std::atomic_store(&topology_, topology::TopologyBase::Create(settings, resolver_)); } -ResultSet ClusterImpl::Execute(engine::Deadline deadline, ClusterHostTypeFlags flags, const Query& query) { - return ExecuteImpl(MergeWithInheritedDeadline(deadline), flags, query); +ResultSet ClusterImpl::Execute( + ClusterHostTypeFlags flags, + OptionalCommandControl command_control, + const Query& query, + const impl::ParameterList& parameters +) { + return ExecuteImpl( + GetExecuteDeadline(ResolveNetworkTimeout(command_control)), + GetExecuteDeadline(ResolveStatementTimeout(command_control)), + flags, + query, + parameters + ); } ResultSet ClusterImpl::ExecuteImpl( - engine::Deadline effective_deadline, + engine::Deadline acquire_deadline, + engine::Deadline statement_deadline, ClusterHostTypeFlags flags, - const Query& query + const Query& query, + const impl::ParameterList& parameters ) { - CheckDeadlineNotExpired(effective_deadline); + CheckDeadlineNotExpired(acquire_deadline); tracing::Span span{storages::odbc::impl::tracing::kExecuteSpan}; - auto& pool = SelectPool(flags); - auto conn = pool.Acquire(effective_deadline); + const auto topology = std::atomic_load(&topology_); + auto& pool = SelectPool(*topology, flags); + auto conn = pool.Acquire(acquire_deadline); pool.AccountOutOfTransaction(); + CheckDeadlineNotExpired(statement_deadline); const auto start = utils::datetime::SteadyCoarseClock::now(); try { - auto result = conn->Query(query.GetStatementView(), effective_deadline); + auto result = conn->Query(query.GetStatementView(), parameters, std::min(acquire_deadline, statement_deadline)); const auto elapsed = std::chrono::duration_cast< std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start); pool.AccountQueryExecuted(elapsed); @@ -60,36 +79,135 @@ ResultSet ClusterImpl::ExecuteImpl( } } -Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags) { - return BeginImpl(GetExecuteDeadline(kDefaultStatementTimeout), flags); -} - -Transaction ClusterImpl::Begin(engine::Deadline deadline, ClusterHostTypeFlags flags) { - return BeginImpl(MergeWithInheritedDeadline(deadline), flags); -} +Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags) { return Begin(flags, std::nullopt); } -Transaction ClusterImpl::BeginImpl(engine::Deadline effective_deadline, ClusterHostTypeFlags flags) { - CheckDeadlineNotExpired(effective_deadline); +Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control) { + const auto acquire_deadline = GetExecuteDeadline(ResolveNetworkTimeout(command_control)); + CheckDeadlineNotExpired(acquire_deadline); tracing::Span span{storages::odbc::impl::tracing::kTransactionSpan}; - auto& pool = SelectPool(flags); - return Transaction{pool.Acquire(effective_deadline), pool, effective_deadline}; + const auto topology = std::atomic_load(&topology_); + auto& pool = SelectPool(*topology, flags); + auto connection = pool.Acquire(acquire_deadline); + const auto statement_deadline = GetExecuteDeadline(ResolveStatementTimeout(command_control)); + return Transaction{std::move(connection), pool, std::min(acquire_deadline, statement_deadline)}; } -Pool& ClusterImpl::SelectPool(ClusterHostTypeFlags flags) const { - UASSERT(topology_); +std::chrono::milliseconds ClusterImpl::ResolveNetworkTimeout(OptionalCommandControl command_control) const { + if (command_control && command_control->network_timeout) { + return *command_control->network_timeout; + } + if (const auto configured = GetDefaultNetworkTimeout()) { + return *configured; + } + return kDefaultStatementTimeout; +} + +std::chrono::milliseconds ClusterImpl::ResolveStatementTimeout(OptionalCommandControl command_control) const { + if (command_control && command_control->statement_timeout) { + return *command_control->statement_timeout; + } + if (const auto configured = GetDefaultStatementTimeout()) { + return *configured; + } + return kDefaultStatementTimeout; +} +Pool& ClusterImpl::SelectPool(const topology::TopologyBase& topology, ClusterHostTypeFlags flags) { if (flags & ClusterHostType::kSlave) { - return topology_->SelectPool(ClusterHostType::kSlave); + return topology.SelectPool(ClusterHostType::kSlave); } // kMaster + kNone go to primary - return topology_->SelectPool(ClusterHostType::kMaster); + return topology.SelectPool(ClusterHostType::kMaster); } void ClusterImpl::WriteStatistics(utils::statistics::Writer& writer) const { - UASSERT(topology_); - topology_->WriteStatistics(writer); + const auto topology = std::atomic_load(&topology_); + UASSERT(topology); + topology->WriteStatistics(writer); +} + +void ClusterImpl::UpdateSettings(const settings::ODBCClusterSettings& settings) { + UINVARIANT(!settings.pools.empty(), "Pools count should be positive"); + bool updated = false; + { + const std::lock_guard lock{settings_mutex_}; + baseline_settings_ = std::make_shared(settings); + updated = UpdateSettingsLocked(MakeEffectiveSettingsLocked()); + } + if (updated) { + TESTPOINT("odbc-new-dsn-list", {}); + } +} + +bool ClusterImpl::UpdateSettingsLocked(const settings::ODBCClusterSettings& settings) { + if (settings_ && *settings_ == settings) { + return false; + } + + // Construct and initialize all new pools before publishing. If this throws, + // the currently working topology remains untouched. + auto new_topology = topology::TopologyBase::Create(settings, resolver_); + auto new_settings = std::make_shared(settings); + settings_ = std::move(new_settings); + std::atomic_store(&topology_, std::move(new_topology)); + return true; +} + +void ClusterImpl::UpdateDsns(const std::vector& dsns) { + UINVARIANT(!dsns.empty(), "ODBC DSN list must not be empty"); + bool updated = false; + { + const std::lock_guard lock{settings_mutex_}; + UASSERT(baseline_settings_ && !baseline_settings_->pools.empty()); + + const bool dsns_unchanged = + dsns.size() == baseline_settings_->pools.size() && + std::equal( + dsns.begin(), + dsns.end(), + baseline_settings_->pools.begin(), + [](const std::string& dsn, const settings::HostSettings& host) { return dsn == host.dsn; } + ); + + if (!dsns_unchanged) { + const auto pool_settings = baseline_settings_->pools.front().pool; + settings::ODBCClusterSettings updated_baseline; + updated_baseline.pools.reserve(dsns.size()); + for (const auto& dsn : dsns) { + updated_baseline.pools.emplace_back(settings::HostSettings{.dsn = dsn, .pool = pool_settings}); + } + baseline_settings_ = std::make_shared(std::move(updated_baseline)); + } + updated = UpdateSettingsLocked(MakeEffectiveSettingsLocked()); + } + if (updated) { + TESTPOINT("odbc-new-dsn-list", {}); + } +} + +void ClusterImpl::SetPoolSettingsOverride(std::optional pool_settings) { + { + const std::lock_guard lock{settings_mutex_}; + if (pool_settings_override_ != pool_settings) { + pool_settings_override_ = pool_settings; + } + UpdateSettingsLocked(MakeEffectiveSettingsLocked()); + } +} + +settings::ODBCClusterSettings ClusterImpl::MakeEffectiveSettingsLocked() const { + UASSERT(baseline_settings_); + settings::ODBCClusterSettings effective; + effective.pools.reserve(baseline_settings_->pools.size()); + for (const auto& host : baseline_settings_->pools) { + effective.pools.emplace_back(settings::HostSettings{ + .dsn = host.dsn, + .pool = pool_settings_override_.value_or(host.pool), + }); + } + return effective; } void ClusterImpl::SetDefaultCommandControl(const CommandControl& cc) { diff --git a/odbc/src/storages/odbc/detail/cluster_impl.hpp b/odbc/src/storages/odbc/detail/cluster_impl.hpp index 80ac6f5000cb..93f72b328cdf 100644 --- a/odbc/src/storages/odbc/detail/cluster_impl.hpp +++ b/odbc/src/storages/odbc/detail/cluster_impl.hpp @@ -9,10 +9,13 @@ #include #include +#include #include #include #include +#include +#include #include #include #include @@ -23,10 +26,6 @@ USERVER_NAMESPACE_BEGIN -namespace storages::odbc { -struct CommandControl; -} - namespace storages::odbc::detail { class ClusterImpl { @@ -35,30 +34,51 @@ class ClusterImpl { ~ClusterImpl() = default; - ResultSet Execute(ClusterHostTypeFlags flags, const Query& query); - - ResultSet Execute(engine::Deadline deadline, ClusterHostTypeFlags flags, const Query& query); + ResultSet Execute( + ClusterHostTypeFlags flags, + OptionalCommandControl command_control, + const Query& query, + const impl::ParameterList& parameters + ); Transaction Begin(ClusterHostTypeFlags flags); - Transaction Begin(engine::Deadline deadline, ClusterHostTypeFlags flags); + Transaction Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control); void WriteStatistics(utils::statistics::Writer& writer) const; void SetDefaultCommandControl(const CommandControl& cc); + void UpdateSettings(const settings::ODBCClusterSettings& settings); + void UpdateDsns(const std::vector& dsns); + void SetPoolSettingsOverride(std::optional settings); + std::optional GetDefaultNetworkTimeout() const; std::optional GetDefaultStatementTimeout() const; private: - Pool& SelectPool(ClusterHostTypeFlags flags) const; - - ResultSet ExecuteImpl(engine::Deadline effective_deadline, ClusterHostTypeFlags flags, const Query& query); - - Transaction BeginImpl(engine::Deadline effective_deadline, ClusterHostTypeFlags flags); - - std::unique_ptr topology_; + static Pool& SelectPool(const topology::TopologyBase& topology, ClusterHostTypeFlags flags); + + ResultSet ExecuteImpl( + engine::Deadline acquire_deadline, + engine::Deadline statement_deadline, + ClusterHostTypeFlags flags, + const Query& query, + const impl::ParameterList& parameters + ); + + std::chrono::milliseconds ResolveNetworkTimeout(OptionalCommandControl command_control) const; + std::chrono::milliseconds ResolveStatementTimeout(OptionalCommandControl command_control) const; + bool UpdateSettingsLocked(const settings::ODBCClusterSettings& settings); + settings::ODBCClusterSettings MakeEffectiveSettingsLocked() const; + + clients::dns::Resolver* resolver_; + std::shared_ptr topology_; + mutable engine::Mutex settings_mutex_; + std::shared_ptr settings_; + std::shared_ptr baseline_settings_; + std::optional pool_settings_override_; // Dynamic config: command control (timeouts) std::atomic default_network_timeout_ms_{std::chrono::milliseconds::zero()}; diff --git a/odbc/src/storages/odbc/detail/connection.cpp b/odbc/src/storages/odbc/detail/connection.cpp index 3b83748e3d12..31802ba00835 100644 --- a/odbc/src/storages/odbc/detail/connection.cpp +++ b/odbc/src/storages/odbc/detail/connection.cpp @@ -1,7 +1,10 @@ #include +#include #include #include +#include +#include #include #include @@ -61,12 +64,219 @@ Connection::DatabaseHandle MakeDatabaseHandle(SQLHENV env) { return Connection::DatabaseHandle(dbc, &DestroyDatabaseHandle); } +struct ParameterBinding final { + SQLSMALLINT c_type; + SQLSMALLINT sql_type; + SQLULEN column_size; + SQLPOINTER data; + SQLLEN buffer_size; +}; + +struct BoundParameter final { + using Value = std::variant; + + explicit BoundParameter(const impl::Parameter& parameter) + : type{parameter.GetType()}, + is_null{parameter.IsNull()}, + value{MakeValue(parameter)} + {} + + static Value MakeValue(const impl::Parameter& parameter) { + using impl::ParameterType; + switch (parameter.GetType()) { + case ParameterType::kBoolean: + return static_cast(parameter.Get() ? 1 : 0); + case ParameterType::kSignedInteger: + return static_cast(parameter.Get()); + case ParameterType::kUnsignedInteger: + return static_cast(parameter.Get()); + case ParameterType::kFloatingPoint: + return static_cast(parameter.Get()); + case ParameterType::kString: + case ParameterType::kUnknown: + return parameter.Get(); + } + UINVARIANT(false, "Unknown ODBC parameter type"); + } + + impl::ParameterType type; + bool is_null; + Value value; +}; + +ParameterBinding GetParameterBinding(BoundParameter& parameter) { + using impl::ParameterType; + + switch (parameter.type) { + case ParameterType::kBoolean: + return { + SQL_C_BIT, + SQL_BIT, + 1, + &std::get(parameter.value), + static_cast(sizeof(SQLCHAR)), + }; + case ParameterType::kSignedInteger: + return { + SQL_C_SBIGINT, + SQL_BIGINT, + 19, + &std::get(parameter.value), + static_cast(sizeof(SQLBIGINT)), + }; + case ParameterType::kUnsignedInteger: + return { + SQL_C_UBIGINT, + SQL_BIGINT, + 20, + &std::get(parameter.value), + static_cast(sizeof(SQLUBIGINT)), + }; + case ParameterType::kFloatingPoint: + return { + SQL_C_DOUBLE, + SQL_DOUBLE, + 15, + &std::get(parameter.value), + static_cast(sizeof(SQLDOUBLE)), + }; + case ParameterType::kString: { + auto& string = std::get(parameter.value); + return { + SQL_C_CHAR, + SQL_VARCHAR, + std::max(1, static_cast(string.size())), + string.data(), + static_cast(string.size()), + }; + } + case ParameterType::kUnknown: + // SQLDescribeParam below replaces the SQL type. A dummy character + // buffer keeps drivers that validate ValuePtr happy for NULL. + return { + SQL_C_CHAR, + SQL_VARCHAR, + 1, + std::get(parameter.value).data(), + 0, + }; + } + UINVARIANT(false, "Unknown ODBC parameter type"); +} + +void BindParameters(SQLHSTMT statement, const impl::ParameterList& parameters, engine::Deadline deadline) { + SQLSMALLINT expected_count = 0; + const auto count_result = SQLNumParams(statement, &expected_count); + if (!SQL_SUCCEEDED(count_result)) { + throw StatementError( + "Failed to determine ODBC parameter count:" + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) + ); + } + if (static_cast(expected_count) != parameters.size()) { + throw StatementError( + fmt::format("ODBC parameter count mismatch: query expects {}, got {}", expected_count, parameters.size()) + ); + } + + std::vector bound_parameters; + bound_parameters.reserve(parameters.size()); + for (const auto& parameter : parameters) { + bound_parameters.emplace_back(parameter); + } + + std::vector indicators(bound_parameters.size()); + for (std::size_t index = 0; index < bound_parameters.size(); ++index) { + auto& parameter = bound_parameters[index]; + auto binding = GetParameterBinding(parameter); + + if (parameter.type == impl::ParameterType::kUnknown) { + SQLSMALLINT decimal_digits = 0; + SQLSMALLINT nullable = SQL_NULLABLE_UNKNOWN; + const auto describe_result = SQLDescribeParam( + statement, + static_cast(index + 1), + &binding.sql_type, + &binding.column_size, + &decimal_digits, + &nullable + ); + if (!SQL_SUCCEEDED(describe_result)) { + throw StatementError( + "Cannot infer the type of a NULL ODBC parameter; add an explicit SQL cast or use a typed " + "std::optional where supported by the driver:" + + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) + ); + } + } + + indicators[index] = parameter.is_null ? SQL_NULL_DATA : binding.buffer_size; + const auto bind_result = SQLBindParameter( + statement, + static_cast(index + 1), + SQL_PARAM_INPUT, + binding.c_type, + binding.sql_type, + binding.column_size, + 0, + binding.data, + binding.buffer_size, + &indicators[index] + ); + if (!SQL_SUCCEEDED(bind_result)) { + throw StatementError( + fmt::format("Failed to bind ODBC parameter {}:", index + 1) + + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) + ); + } + } + + detail::CheckDeadlineNotExpired(deadline); + const auto execute_result = SQLExecute(statement); + detail::CheckDeadlineNotExpired(deadline); + if (!SQL_SUCCEEDED(execute_result) && execute_result != SQL_NO_DATA) { + throw StatementError( + "Failed to execute prepared ODBC query:" + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) + ); + } +} + +SQLRETURN ExecuteStatement( + SQLHSTMT statement, + std::string_view query, + const impl::ParameterList& parameters, + engine::Deadline deadline +) { + std::vector query_buffer(query.begin(), query.end()); + query_buffer.push_back('\0'); + + if (parameters.empty()) { + detail::CheckDeadlineNotExpired(deadline); + const auto execute_result = SQLExecDirect(statement, query_buffer.data(), SQL_NTS); + detail::CheckDeadlineNotExpired(deadline); + return execute_result; + } + + detail::CheckDeadlineNotExpired(deadline); + const auto prepare_result = SQLPrepare(statement, query_buffer.data(), SQL_NTS); + detail::CheckDeadlineNotExpired(deadline); + if (!SQL_SUCCEEDED(prepare_result)) { + throw StatementError("Failed to prepare ODBC query:" + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT)); + } + BindParameters(statement, parameters, deadline); + return SQL_SUCCESS; +} + } // namespace Connection::Connection(const std::string& dsn) + : Connection{dsn, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)} +{} + +Connection::Connection(const std::string& dsn, engine::Deadline deadline) : env_(MakeEnvironmentHandle()), handle_(Connection::DatabaseHandle(SQL_NULL_HDBC, &DestroyDatabaseHandle)) { + detail::CheckDeadlineNotExpired(deadline); SQLRETURN ret = SQLSetEnvAttr(env_.get(), SQL_ATTR_CONNECTION_POOLING, reinterpret_cast(SQL_CP_ONE_PER_DRIVER), 0); if (!SQL_SUCCEEDED(ret)) { @@ -82,16 +292,37 @@ Connection::Connection(const std::string& dsn) handle_ = MakeDatabaseHandle(env_.get()); + if (deadline.IsReachable()) { + const auto time_left = deadline.TimeLeft(); + if (time_left <= engine::Deadline::Duration::zero()) { + detail::CheckDeadlineNotExpired(deadline); + } + const auto timeout = std::chrono::ceil(time_left); + const auto timeout_seconds = static_cast(timeout.count()); + ret = SQLSetConnectAttr( + handle_.get(), + SQL_ATTR_LOGIN_TIMEOUT, + reinterpret_cast(static_cast(timeout_seconds)), + SQL_IS_UINTEGER + ); + if (!SQL_SUCCEEDED(ret)) { + throw ConnectionError( + "Failed to set ODBC login timeout:" + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) + ); + } + } + std::vector dsn_buffer(dsn.begin(), dsn.end()); dsn_buffer.push_back('\0'); + detail::CheckDeadlineNotExpired(deadline); ret = SQLDriverConnect(handle_.get(), nullptr, dsn_buffer.data(), SQL_NTS, nullptr, 0, nullptr, SQL_DRIVER_COMPLETE); + detail::CheckDeadlineNotExpired(deadline); if (!SQL_SUCCEEDED(ret)) { throw ConnectionError( "Failed to connect to database: " + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) ); } - SQLUINTEGER scroll_option = 0; ret = SQLGetInfo(handle_.get(), SQL_SCROLL_OPTIONS, &scroll_option, sizeof(scroll_option), nullptr); if (!SQL_SUCCEEDED(ret)) { @@ -107,10 +338,18 @@ Connection::Connection(const std::string& dsn) } ResultSet Connection::Query(std::string_view query) { - return Query(query, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)); + return Query(query, impl::ParameterList{}, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)); } ResultSet Connection::Query(std::string_view query, engine::Deadline deadline) { + return Query(query, impl::ParameterList{}, deadline); +} + +ResultSet Connection::Query(std::string_view query, const impl::ParameterList& parameters) { + return Query(query, parameters, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)); +} + +ResultSet Connection::Query(std::string_view query, const impl::ParameterList& parameters, engine::Deadline deadline) { detail::CheckDeadlineNotExpired(deadline); auto guard = GetBrokenGuard(); @@ -123,25 +362,27 @@ ResultSet Connection::Query(std::string_view query, engine::Deadline deadline) { if (deadline.IsReachable()) { const auto left = deadline.TimeLeft(); - if (left > std::chrono::milliseconds::zero()) { - const auto seconds = std::chrono::ceil(left); - const auto timeout_sec = static_cast(seconds.count()); - if (timeout_sec > 0) { - /* ODBC SQL_ATTR_QUERY_TIMEOUT is in whole seconds; deadline checks still use full TimeLeft() - * resolution. */ - SQLSetStmtAttr( - stmt.get(), - SQL_ATTR_QUERY_TIMEOUT, - reinterpret_cast(static_cast(timeout_sec)), - 0 - ); - } + if (left <= engine::Deadline::Duration::zero()) { + detail::CheckDeadlineNotExpired(deadline); + } + const auto seconds = std::chrono::ceil(left); + const auto timeout_sec = static_cast(seconds.count()); + /* ODBC SQL_ATTR_QUERY_TIMEOUT is in whole seconds; deadline checks still use full TimeLeft() + * resolution. */ + const auto timeout_result = SQLSetStmtAttr( + stmt.get(), + SQL_ATTR_QUERY_TIMEOUT, + reinterpret_cast(static_cast(timeout_sec)), + 0 + ); + if (!SQL_SUCCEEDED(timeout_result)) { + throw StatementError( + "Failed to set ODBC query timeout:" + detail::GetSQLDiagString(stmt.get(), SQL_HANDLE_STMT) + ); } } - std::vector query_buffer(query.begin(), query.end()); - query_buffer.push_back('\0'); - SQLRETURN ret = SQLExecDirect(stmt.get(), query_buffer.data(), SQL_NTS); + SQLRETURN ret = ExecuteStatement(stmt.get(), query, parameters, deadline); if (!SQL_SUCCEEDED(ret) && ret != SQL_NO_DATA) { const auto diag = detail::GetSQLDiagString(stmt.get(), SQL_HANDLE_STMT); span.AddTag(tracing::kErrorFlag, true); @@ -149,7 +390,6 @@ ResultSet Connection::Query(std::string_view query, engine::Deadline deadline) { throw StatementError("Failed to execute query:" + diag); } - auto wrapper = std::make_shared(std::move(stmt)); // Only call Fetch for SELECT-like statements that produce a result set. // DML statements (INSERT/UPDATE/DELETE) have 0 result columns; calling // SQLFetch on them returns SQL_NO_DATA or an error depending on the driver. @@ -157,11 +397,13 @@ ResultSet Connection::Query(std::string_view query, engine::Deadline deadline) { SQLSMALLINT col_count = 0; SQLNumResultCols(stmt.get(), &col_count); if (col_count > 0) { + auto wrapper = std::make_shared(std::move(stmt)); wrapper->Fetch(); + return ResultSet(std::move(wrapper)); } } - return ResultSet(std::move(wrapper)); + return ResultSet(std::make_shared(std::move(stmt))); }); } diff --git a/odbc/src/storages/odbc/detail/connection.hpp b/odbc/src/storages/odbc/detail/connection.hpp index cff94247d30b..255ac899b9a2 100644 --- a/odbc/src/storages/odbc/detail/connection.hpp +++ b/odbc/src/storages/odbc/detail/connection.hpp @@ -10,6 +10,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -27,6 +28,7 @@ class Connection final { using DatabaseHandle = std::unique_ptr, void (*)(SQLHDBC)>; explicit Connection(const std::string& dsn); + Connection(const std::string& dsn, engine::Deadline deadline); ~Connection() = default; @@ -38,9 +40,14 @@ class Connection final { /// @return ResultSet containing the query results ResultSet Query(std::string_view query); + /// @brief Executes a prepared SQL query with separately bound parameters. + ResultSet Query(std::string_view query, const impl::ParameterList& parameters); + /// @brief Same as Query(std::string_view), but honours \a deadline for wait / driver timeout. ResultSet Query(std::string_view query, engine::Deadline deadline); + ResultSet Query(std::string_view query, const impl::ParameterList& parameters, engine::Deadline deadline); + // required by ConnectionPool bool IsBroken() const; void NotifyBroken(); diff --git a/odbc/src/storages/odbc/detail/pool.cpp b/odbc/src/storages/odbc/detail/pool.cpp index f0d02f43d89d..cbeab96b3faa 100644 --- a/odbc/src/storages/odbc/detail/pool.cpp +++ b/odbc/src/storages/odbc/detail/pool.cpp @@ -1,8 +1,10 @@ #include +#include #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -22,11 +24,12 @@ Pool::Pool(const std::string& dsn, std::size_t min_pool_size, std::size_t max_po stats_.connection.maximum = max_pool_size; try { Init(min_pool_size, kInitTimeout); - } catch (const Error& odbc_err) { - Reset(); - throw; } catch (const std::exception& ex) { LOG_ERROR() << "Error while initializing ODBC connection pool: " << ex; + Reset(); + throw; + } catch (...) { + Reset(); throw; } } @@ -39,11 +42,12 @@ Pool::Pool(std::vector dsns, std::size_t min_pool_size, std::size_t stats_.connection.maximum = max_pool_size; try { Init(min_pool_size, kInitTimeout); - } catch (const Error& odbc_err) { - Reset(); - throw; } catch (const std::exception& ex) { LOG_ERROR() << "Error while initializing ODBC connection pool: " << ex; + Reset(); + throw; + } catch (...) { + Reset(); throw; } } @@ -53,10 +57,10 @@ Pool::~Pool() { Reset(); } ConnectionPtr Pool::Acquire(engine::Deadline deadline) { const auto start = utils::datetime::SteadyCoarseClock::now(); ++stats_.connection.waiting; + const utils::FastScopeGuard waiting_guard([this]() noexcept { --stats_.connection.waiting; }); auto conn_wrapper = AcquireConnection(deadline); - --stats_.connection.waiting; ++stats_.connection.used; const auto elapsed = std::chrono::duration_cast< @@ -72,15 +76,11 @@ void Pool::Release(ConnectionUniquePtr connection) { } Pool::ConnectionUniquePtr Pool::DoCreateConnection(engine::Deadline deadline) { - if (deadline.IsReached()) { - ++stats_.connection.error_timeout; - throw std::runtime_error("Connection creation deadline reached"); - } - const auto start = utils::datetime::SteadyCoarseClock::now(); try { + CheckDeadlineNotExpired(deadline); const auto idx = dsn_index_.fetch_add(1); - auto conn = std::make_unique(dsns_[idx % dsns_.size()]); + auto conn = std::make_unique(dsns_[idx % dsns_.size()], deadline); ++stats_.connection.open_total; const auto elapsed = std::chrono::duration_cast< @@ -88,6 +88,10 @@ Pool::ConnectionUniquePtr Pool::DoCreateConnection(engine::Deadline deadline) { stats_.connection_percentile.Account(elapsed.count()); return conn; + } catch (const OperationInterrupted& ex) { + ++stats_.connection.error_timeout; + LOG_ERROR() << "Timed out while creating ODBC connection: " << ex; + throw; } catch (const std::exception& ex) { ++stats_.connection.error_total; LOG_ERROR() << "Failed to create ODBC connection: " << ex; diff --git a/odbc/src/storages/odbc/detail/topology/topology_base.cpp b/odbc/src/storages/odbc/detail/topology/topology_base.cpp index 8a5a9d2cf48c..10dd48944a03 100644 --- a/odbc/src/storages/odbc/detail/topology/topology_base.cpp +++ b/odbc/src/storages/odbc/detail/topology/topology_base.cpp @@ -38,17 +38,17 @@ TopologyBase::TopologyBase(const settings::ODBCClusterSettings& settings, client TopologyBase::~TopologyBase() = default; -std::unique_ptr TopologyBase::Create( +std::shared_ptr TopologyBase::Create( const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver ) { UASSERT(!settings.pools.empty()); if (settings.pools.size() == 1) { - return std::make_unique(settings, resolver); + return std::make_shared(settings, resolver); } - return std::make_unique(settings, resolver); + return std::make_shared(settings, resolver); } Pool& TopologyBase::SelectPool(ClusterHostType host_type) const { diff --git a/odbc/src/storages/odbc/detail/topology/topology_base.hpp b/odbc/src/storages/odbc/detail/topology/topology_base.hpp index e741151eae94..d4c710a5af90 100644 --- a/odbc/src/storages/odbc/detail/topology/topology_base.hpp +++ b/odbc/src/storages/odbc/detail/topology/topology_base.hpp @@ -22,7 +22,7 @@ class TopologyBase { public: virtual ~TopologyBase(); - static std::unique_ptr Create( + static std::shared_ptr Create( const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver ); diff --git a/odbc/src/storages/odbc/odbc_config.hpp b/odbc/src/storages/odbc/odbc_config.hpp deleted file mode 100644 index ea0e491171c0..000000000000 --- a/odbc/src/storages/odbc/odbc_config.hpp +++ /dev/null @@ -1,18 +0,0 @@ -#pragma once - -#include -#include -#include - -USERVER_NAMESPACE_BEGIN - -namespace storages::odbc { - -struct CommandControl { - std::optional network_timeout; - std::optional statement_timeout; -}; - -} // namespace storages::odbc - -USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/odbc_secdist.cpp b/odbc/src/storages/odbc/odbc_secdist.cpp index 50048e9bbabf..df718b276f34 100644 --- a/odbc/src/storages/odbc/odbc_secdist.cpp +++ b/odbc/src/storages/odbc/odbc_secdist.cpp @@ -70,6 +70,13 @@ OdbcSettings::OdbcSettings(const formats::json::Value& doc) { if (connections.empty()) { throw storages::secdist::SecdistError(fmt::format("Database '{}' has no connection info", dbalias)); } + for (std::size_t index = 0; index < connections.size(); ++index) { + if (connections[index].dsn.empty()) { + throw storages::secdist::SecdistError( + fmt::format("Database '{}' has an empty DSN at connection {}", dbalias, index) + ); + } + } databases_[dbalias] = std::move(connections); } diff --git a/odbc/src/storages/odbc/transaction.cpp b/odbc/src/storages/odbc/transaction.cpp index 64d3612965f6..67b5e6defbb1 100644 --- a/odbc/src/storages/odbc/transaction.cpp +++ b/odbc/src/storages/odbc/transaction.cpp @@ -1,5 +1,6 @@ #include +#include #include #include #include @@ -65,14 +66,31 @@ void Transaction::Rollback() { pool_->AccountTransactionRollback(); } -ResultSet Transaction::Execute(const Query& query) { +ResultSet Transaction::DoExecute( + OptionalCommandControl command_control, + const Query& query, + const impl::ParameterList& parameters +) { AssertValid(); detail::CheckDeadlineNotExpired(deadline_); tracing::Span span{storages::odbc::impl::tracing::kExecuteSpan}; + auto statement_deadline = deadline_; + if (command_control) { + if (command_control->network_timeout) { + statement_deadline = + std::min(statement_deadline, detail::GetExecuteDeadline(*command_control->network_timeout)); + } + if (command_control->statement_timeout) { + statement_deadline = + std::min(statement_deadline, detail::GetExecuteDeadline(*command_control->statement_timeout)); + } + } + detail::CheckDeadlineNotExpired(statement_deadline); + const auto start = utils::datetime::SteadyCoarseClock::now(); try { - auto result = (*connection_)->Query(query.GetStatementView(), deadline_); + auto result = (*connection_)->Query(query.GetStatementView(), parameters, statement_deadline); const auto elapsed = std::chrono::duration_cast< std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start); busy_time_ += elapsed; diff --git a/odbc/tests/odbc_deadline_test.cpp b/odbc/tests/odbc_deadline_test.cpp index 02500933313f..10580244152f 100644 --- a/odbc/tests/odbc_deadline_test.cpp +++ b/odbc/tests/odbc_deadline_test.cpp @@ -38,7 +38,11 @@ UTEST(OdbcDeadline, InheritedExpiredOverridesLongExplicitExecute) { server::request::kTaskInheritedData.Set(MakeRequestData(engine::Deadline::Passed())); UEXPECT_THROW( - cluster.Execute(engine::Deadline::FromDuration(1h), storages::odbc::ClusterHostType::kMaster, "SELECT 1"), + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{.statement_timeout = 1h}, + "SELECT 1" + ), storages::odbc::OperationInterrupted ); } @@ -47,7 +51,25 @@ UTEST(OdbcDeadline, ExplicitExpiredExecute) { auto cluster = MakeCluster(); UEXPECT_THROW( - cluster.Execute(engine::Deadline::Passed(), storages::odbc::ClusterHostType::kMaster, "SELECT 1"), + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{.statement_timeout = 0ms}, + "SELECT 1" + ), + storages::odbc::OperationInterrupted + ); +} + +UTEST(OdbcDeadline, ExpiresDuringBlockingExecute) { + auto cluster = MakeCluster(); + + UEXPECT_THROW( + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{.statement_timeout = 1ms}, + "SELECT pg_sleep(CAST(? AS double precision))", + 0.05 + ), storages::odbc::OperationInterrupted ); } diff --git a/odbc/tests/odbc_postgresql_test.cpp b/odbc/tests/odbc_postgresql_test.cpp index d0c38c1ce796..a3228b2c7480 100644 --- a/odbc/tests/odbc_postgresql_test.cpp +++ b/odbc/tests/odbc_postgresql_test.cpp @@ -1,4 +1,7 @@ #include +#include +#include +#include #include #include #include @@ -34,6 +37,58 @@ UTEST(Query, Works) { } } +UTEST(Query, BindsParametersWithoutInterpolation) { + auto cluster = MakeCluster(); + + /// [ODBC parameter binding] + const std::string untrusted_value = "Robert'); DROP TABLE users;--"; + const auto result = cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "SELECT ?::text, ?::text, ?::bigint, ?::bigint, ?::double precision, ?::boolean, ?::boolean", + untrusted_value, + std::string_view{""}, + std::int16_t{-42}, + std::uint32_t{42}, + 1.25F, + true, + false + ); + /// [ODBC parameter binding] + + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetString(), untrusted_value); + EXPECT_EQ(result[0][1].GetString(), ""); + EXPECT_EQ(result[0][2].GetInt64(), -42); + EXPECT_EQ(result[0][3].GetInt64(), 42); + EXPECT_DOUBLE_EQ(result[0][4].GetDouble(), 1.25); + EXPECT_TRUE(result[0][5].GetBool()); + EXPECT_FALSE(result[0][6].GetBool()); +} + +UTEST(Query, BindsTypedNull) { + auto cluster = MakeCluster(); + + const std::optional value; + const auto result = cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "SELECT ?::text IS NULL, ?::text IS NULL", + value, + nullptr + ); + + ASSERT_EQ(result.Size(), 1); + EXPECT_TRUE(result[0][0].GetBool()); + EXPECT_TRUE(result[0][1].GetBool()); +} + +UTEST(Query, ParameterCountMismatchIsStatementError) { + auto cluster = MakeCluster(); + UEXPECT_THROW( + cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELECT ?::integer", 1, 2), + storages::odbc::StatementError + ); +} + UTEST(Query, VariousTypes) { auto query = "SELECT 42, 'test', 1.0, false, null, true"; auto cluster = MakeCluster(); diff --git a/odbc/tests/odbc_transaction_test.cpp b/odbc/tests/odbc_transaction_test.cpp index f3c9c3df02fd..28742d1eca4e 100644 --- a/odbc/tests/odbc_transaction_test.cpp +++ b/odbc/tests/odbc_transaction_test.cpp @@ -40,6 +40,20 @@ UTEST(Transaction, QueryInTransaction) { trx.Commit(); } +UTEST(Transaction, BindsParameters) { + auto cluster = MakeCluster(); + auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); + + /// [ODBC transaction parameter binding] + const auto result = trx.Execute("SELECT ?::text, ?::integer", "quoted ' value", 42); + /// [ODBC transaction parameter binding] + + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetString(), "quoted ' value"); + EXPECT_EQ(result[0][1].GetInt32(), 42); + trx.Commit(); +} + UTEST(Transaction, DoubleCommitThrows) { auto cluster = MakeCluster(); auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); diff --git a/scripts/docs/en/userver/odbc.md b/scripts/docs/en/userver/odbc.md index d21a0cbb0b5b..c9dedb92f427 100644 --- a/scripts/docs/en/userver/odbc.md +++ b/scripts/docs/en/userver/odbc.md @@ -1,8 +1,94 @@ ## ODBC Driver -🐙 **userver** provides generic asynchronous driver for SQL-like databases via ODBC. The driver is in early -development stage and can only execute simple queries. +🐙 **userver** provides an asynchronous client for SQL databases that expose an +ODBC interface. The driver integrates connection pools, transactions, +deadlines, tracing, metrics, dynamic configuration, secdist and DNS resolution +with the userver component system. +### Executing queries safely + +Use `?` placeholders and pass values separately. The driver prepares the SQL +statement and passes every value separately to the ODBC binding API with +`SQLBindParameter`. userver does not interpolate values into SQL: the selected +ODBC driver handles their escaping and typing, so an untrusted value cannot +alter the query structure. A driver may still serialize bound values as SQL +literals internally. + +@snippet odbc/tests/odbc_postgresql_test.cpp ODBC parameter binding + +The variadic API supports booleans, signed and unsigned integers, floating +point values, strings and string views. Use `std::optional` for a typed +nullable value, or `nullptr` when the ODBC driver can infer the parameter type +from the statement. The number of C++ arguments must match the number of `?` +placeholders. + +`storages::odbc::Cluster::Execute` returns a storages::odbc::ResultSet. Its rows +contain storages::odbc::Field values that provide typed getters such as +`GetInt32`, `GetInt64`, `GetDouble`, `GetBool`, and `GetString`. + +### Transactions + +Transactions are created with storages::odbc::Cluster::Begin. They commit or +roll back explicitly and automatically roll back on destruction if left open. +Parameters are bound in transaction queries in exactly the same way: + +@snippet odbc/tests/odbc_transaction_test.cpp ODBC transaction parameter binding + +### Command control and deadlines + +storages::odbc::CommandControl configures the connection-acquisition/network +timeout and statement timeout for an operation. Pass an +storages::odbc::OptionalCommandControl to `Cluster::Execute`, `Cluster::Begin`, +or `Transaction::Execute` to override the defaults. The effective deadline is +the earliest of the network operation budget, statement timeout, transaction +deadline, and task-inherited request deadline. + +ODBC `SQL_ATTR_QUERY_TIMEOUT` has whole-second resolution. The driver rounds a +positive sub-second value up when passing it to ODBC while retaining the exact +userver deadline for pool waits and pre-operation checks. Cancellation of a +blocking ODBC call itself depends on timeout support in the selected driver. + +Deadline expiry is reported as storages::odbc::OperationInterrupted. +Connection and driver failures use storages::odbc::ConnectionError, and +statement preparation, binding, and execution failures use +storages::odbc::StatementError. + +### Component configuration + +Add components::Odbc under `components_manager.components`. The following +tested configuration obtains its DSN from secdist: + +@snippet odbc/functional_tests/basic_chaos/static_config.yaml ODBC component config + +The complete generated static-config schema, including the mutually exclusive +`dsn`, `pools`, and `secdist_alias` connection sources, is available on +components::Odbc. + +For secdist, `odbc_settings.databases.` accepts either a `dsn` string or +a `hosts` array. A host can be a DSN string or an object with a `dsn` member. +Using secdist keeps credentials out of the static configuration and supports +live credential/endpoint updates. + +The `dns_resolver` static option selects `async` (the default, userver DNS +resolver) or `getaddrinfo` (resolution in the ODBC driver). + +### Dynamic configuration and metrics + +@ref USERVER_ODBC_DEFAULT_COMMAND_CONTROL controls default network and +statement timeouts. @ref USERVER_ODBC_CONNECTION_POOL_SETTINGS describes pool +settings by component name and the `__default__` fallback. Their schemas and +defaults are generated from the dynamic-config YAML sources and included in +the dynamic-config reference. + +The component exports pool, query, error, timeout, and transaction statistics +under the `odbc` metric prefix, labelled with the component and pool. + +@section odbc_info More information + +- For component options and the generated schema, see components::Odbc. +- For query execution, see storages::odbc::Cluster. +- For result traversal, see storages::odbc::ResultSet. +- For transaction semantics, see storages::odbc::Transaction. ---------- From 25bfb602c8965e1c78828f2b6052399586f5f7c1 Mon Sep 17 00:00:00 2001 From: Andrey Balabekyan <55596549+V0S7ER@users.noreply.github.com> Date: Sun, 2 Aug 2026 02:52:16 +0300 Subject: [PATCH 02/12] fix odbc: harden driver lifecycle --- .../basic_chaos/static_config.yaml | 1 + .../secdist_update/static_config.yaml | 1 + .../include/userver/storages/odbc/cluster.hpp | 6 + .../userver/storages/odbc/exception.hpp | 31 + .../userver/storages/odbc/result_set.hpp | 8 + .../userver/storages/odbc/transaction.hpp | 10 +- odbc/src/storages/odbc/cluster.cpp | 11 +- odbc/src/storages/odbc/component.cpp | 17 +- odbc/src/storages/odbc/component.yaml | 5 + .../src/storages/odbc/detail/broken_guard.cpp | 2 +- .../src/storages/odbc/detail/cluster_impl.cpp | 84 +-- .../src/storages/odbc/detail/cluster_impl.hpp | 21 +- odbc/src/storages/odbc/detail/connection.cpp | 703 +++++++++++++----- odbc/src/storages/odbc/detail/connection.hpp | 21 +- odbc/src/storages/odbc/detail/deadline.cpp | 5 +- odbc/src/storages/odbc/detail/deadline.hpp | 3 + .../src/storages/odbc/detail/diag_wrapper.cpp | 51 +- .../src/storages/odbc/detail/diag_wrapper.hpp | 10 + odbc/src/storages/odbc/detail/pool.cpp | 97 ++- odbc/src/storages/odbc/detail/pool.hpp | 20 +- .../storages/odbc/detail/result_wrapper.cpp | 207 ++---- .../storages/odbc/detail/result_wrapper.hpp | 51 +- odbc/src/storages/odbc/detail/statistics.hpp | 2 +- .../odbc/detail/topology/fixed_primary.cpp | 8 +- .../odbc/detail/topology/fixed_primary.hpp | 6 +- .../odbc/detail/topology/standalone.cpp | 8 +- .../odbc/detail/topology/standalone.hpp | 6 +- .../odbc/detail/topology/topology_base.cpp | 16 +- .../odbc/detail/topology/topology_base.hpp | 10 +- odbc/src/storages/odbc/detail/tracing.cpp | 16 + odbc/src/storages/odbc/detail/tracing.hpp | 10 + odbc/src/storages/odbc/dsn.cpp | 233 ++++-- odbc/src/storages/odbc/dsn.hpp | 7 + odbc/src/storages/odbc/exception.cpp | 22 + odbc/src/storages/odbc/odbc_secdist.cpp | 18 +- odbc/src/storages/odbc/result_set.cpp | 9 + odbc/src/storages/odbc/transaction.cpp | 48 +- odbc/tests/odbc_dsn_test.cpp | 31 + odbc/tests/odbc_exceptions_test.cpp | 84 ++- odbc/tests/odbc_postgresql_test.cpp | 152 ++++ odbc/tests/odbc_round_robin_pool_test.cpp | 3 +- odbc/tests/odbc_transaction_test.cpp | 105 +++ scripts/docs/en/userver/odbc.md | 33 +- 43 files changed, 1591 insertions(+), 601 deletions(-) diff --git a/odbc/functional_tests/basic_chaos/static_config.yaml b/odbc/functional_tests/basic_chaos/static_config.yaml index 7644c252a792..8a38f707ed11 100644 --- a/odbc/functional_tests/basic_chaos/static_config.yaml +++ b/odbc/functional_tests/basic_chaos/static_config.yaml @@ -12,6 +12,7 @@ components_manager: # [ODBC component config] key-value-db: + blocking_task_processor: fs-task-processor secdist_alias: key-value-db min_pool_size: 1 max_pool_size: 1 diff --git a/odbc/functional_tests/secdist_update/static_config.yaml b/odbc/functional_tests/secdist_update/static_config.yaml index 0c5b91ed8b38..ee56379feaf4 100644 --- a/odbc/functional_tests/secdist_update/static_config.yaml +++ b/odbc/functional_tests/secdist_update/static_config.yaml @@ -6,6 +6,7 @@ components_manager: method: GET odbc-database: + blocking_task_processor: fs-task-processor secdist_alias: odbc-test min_pool_size: 0 max_pool_size: 1 diff --git a/odbc/include/userver/storages/odbc/cluster.hpp b/odbc/include/userver/storages/odbc/cluster.hpp index e88403d88df5..785a9d0b923e 100644 --- a/odbc/include/userver/storages/odbc/cluster.hpp +++ b/odbc/include/userver/storages/odbc/cluster.hpp @@ -7,6 +7,7 @@ #include #include +#include #include #include @@ -32,6 +33,11 @@ using ClusterImplPtr = std::unique_ptr; class Cluster { public: Cluster(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver); + Cluster( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor + ); ~Cluster(); diff --git a/odbc/include/userver/storages/odbc/exception.hpp b/odbc/include/userver/storages/odbc/exception.hpp index 20469b994b06..ff8bbaa93484 100644 --- a/odbc/include/userver/storages/odbc/exception.hpp +++ b/odbc/include/userver/storages/odbc/exception.hpp @@ -5,11 +5,21 @@ #include #include +#include +#include +#include USERVER_NAMESPACE_BEGIN namespace storages::odbc { +/// A single diagnostic record reported by the ODBC driver manager or driver. +struct DiagnosticRecord final { + std::string sql_state; + int native_error{0}; + std::string message; +}; + class Error : public std::runtime_error { using std::runtime_error::runtime_error; }; @@ -19,7 +29,23 @@ class LogicError : public Error { }; class RuntimeError : public Error { +public: using Error::Error; + + RuntimeError(std::string message, std::vector diagnostics, bool invalid_handle = false); + + /// Structured driver diagnostics, in the order returned by ODBC. + const std::vector& GetDiagnostics() const noexcept; + + /// Whether any diagnostic has the specified two-character SQLSTATE class. + bool HasSqlStateClass(std::string_view sql_state_class) const noexcept; + + /// Whether the failed ODBC call returned SQL_INVALID_HANDLE. + bool IsInvalidHandle() const noexcept; + +private: + std::vector diagnostics_; + bool invalid_handle_{false}; }; class ConnectionError : public RuntimeError { @@ -30,6 +56,11 @@ class StatementError : public RuntimeError { using RuntimeError::RuntimeError; }; +/// Thrown when an ODBC pool cannot provide a connection for a non-timeout reason. +class PoolError : public RuntimeError { + using RuntimeError::RuntimeError; +}; + /// Thrown when the operation is aborted because an @ref engine::Deadline has expired /// (including task-inherited request deadlines). class OperationInterrupted : public RuntimeError { diff --git a/odbc/include/userver/storages/odbc/result_set.hpp b/odbc/include/userver/storages/odbc/result_set.hpp index 3365d615b0b5..01644a14cbf6 100644 --- a/odbc/include/userver/storages/odbc/result_set.hpp +++ b/odbc/include/userver/storages/odbc/result_set.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -36,6 +37,13 @@ class ResultSet final { size_type Size() const; + /// @brief Number of rows affected by a data-modifying statement. + /// Returns zero when the driver reports an unknown count. + size_type RowsAffected() const; + + /// @brief Get a result column name by zero-based index. + std::string_view GetFieldName(size_type index) const; + /// @brief Check if the result set is empty bool IsEmpty() const; diff --git a/odbc/include/userver/storages/odbc/transaction.hpp b/odbc/include/userver/storages/odbc/transaction.hpp index 002c0b63ff12..268ff068d9f4 100644 --- a/odbc/include/userver/storages/odbc/transaction.hpp +++ b/odbc/include/userver/storages/odbc/transaction.hpp @@ -31,7 +31,12 @@ class Pool; /// storages::odbc::Cluster class Transaction final { public: - explicit Transaction(detail::ConnectionPtr&& connection, detail::Pool& pool, engine::Deadline deadline); + explicit Transaction( + detail::ConnectionPtr&& connection, + detail::Pool& pool, + std::chrono::milliseconds network_timeout, + std::chrono::milliseconds statement_timeout + ); ~Transaction(); Transaction(const Transaction& other) = delete; Transaction(Transaction&& other) noexcept; @@ -65,7 +70,8 @@ class Transaction final { // shared_ptr(16) + unique_ptr(8) = 24 bytes, align 8 utils::FastPimpl connection_; detail::Pool* pool_; - engine::Deadline deadline_; + std::chrono::milliseconds network_timeout_; + std::chrono::milliseconds statement_timeout_; utils::datetime::SteadyCoarseClock::time_point start_time_; std::chrono::microseconds busy_time_{0}; tracing::Span span_; diff --git a/odbc/src/storages/odbc/cluster.cpp b/odbc/src/storages/odbc/cluster.cpp index 1d9a79c40469..de1f73235e2c 100644 --- a/odbc/src/storages/odbc/cluster.cpp +++ b/odbc/src/storages/odbc/cluster.cpp @@ -3,6 +3,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -10,7 +11,15 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc { Cluster::Cluster(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) - : impl_(std::make_unique(settings, resolver)) + : Cluster{settings, resolver, engine::current_task::GetBlockingTaskProcessor()} +{} + +Cluster::Cluster( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor +) + : impl_(std::make_unique(settings, resolver, blocking_task_processor)) { UASSERT(!settings.pools.empty()); } diff --git a/odbc/src/storages/odbc/component.cpp b/odbc/src/storages/odbc/component.cpp index ccb8535e743c..d4f1c800bc55 100644 --- a/odbc/src/storages/odbc/component.cpp +++ b/odbc/src/storages/odbc/component.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,14 @@ void ValidateNonEmpty(std::string_view value, std::string_view option) { } } +engine::TaskProcessor& GetBlockingTaskProcessor( + const components::ComponentConfig& config, + const components::ComponentContext& context +) { + const auto name = config["blocking_task_processor"].As>(); + return name ? context.GetTaskProcessor(*name) : engine::current_task::GetBlockingTaskProcessor(); +} + storages::odbc::settings::ODBCClusterSettings MakeClusterSettingsFromConfig(const components::ComponentConfig& config) { using storages::odbc::settings::HostSettings; using storages::odbc::settings::ODBCClusterSettings; @@ -144,9 +153,11 @@ Odbc::Odbc(const ComponentConfig& config, const ComponentContext& context) : ComponentBase{config, context}, name_{config.Name()}, secdist_alias_{config["secdist_alias"].As>()}, - cluster_{std::make_shared< - storages::odbc::Cluster>(MakeClusterSettings(config, context), clients::dns::GetResolverPtr(config, context)) - }, + cluster_{std::make_shared( + MakeClusterSettings(config, context), + clients::dns::GetResolverPtr(config, context), + GetBlockingTaskProcessor(config, context) + )}, config_source_{context.FindComponent().GetSource()} { utils::statistics::RegisterWriterScope( diff --git a/odbc/src/storages/odbc/component.yaml b/odbc/src/storages/odbc/component.yaml index 066bcb7d2318..8e2e08fb6311 100644 --- a/odbc/src/storages/odbc/component.yaml +++ b/odbc/src/storages/odbc/component.yaml @@ -2,6 +2,11 @@ type: object description: ODBC client component additionalProperties: false properties: + blocking_task_processor: + type: string + description: | + Task processor used for synchronous ODBC driver-manager and driver calls. + Defaults to the global blocking task processor. secdist_alias: type: string description: | diff --git a/odbc/src/storages/odbc/detail/broken_guard.cpp b/odbc/src/storages/odbc/detail/broken_guard.cpp index 32d637561fd5..a53f04e28d2a 100644 --- a/odbc/src/storages/odbc/detail/broken_guard.cpp +++ b/odbc/src/storages/odbc/detail/broken_guard.cpp @@ -10,7 +10,7 @@ BrokenGuard::BrokenGuard(Connection& connection) : connection_{connection}, exceptions_on_enter_{std::uncaught_exceptions()} { - if (connection_.IsBroken()) { + if (connection_.IsMarkedBroken()) { throw ConnectionError("Connection is broken."); } } diff --git a/odbc/src/storages/odbc/detail/cluster_impl.cpp b/odbc/src/storages/odbc/detail/cluster_impl.cpp index c7a29cf8d237..488c56a50e55 100644 --- a/odbc/src/storages/odbc/detail/cluster_impl.cpp +++ b/odbc/src/storages/odbc/detail/cluster_impl.cpp @@ -20,15 +20,18 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail { -static_assert(std::atomic::is_always_lock_free); - -ClusterImpl::ClusterImpl(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) +ClusterImpl::ClusterImpl( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor +) : resolver_{resolver}, + blocking_task_processor_{blocking_task_processor}, settings_{std::make_shared(settings)}, baseline_settings_{settings_} { UINVARIANT(!settings.pools.empty(), "Pools count should be positive"); - std::atomic_store(&topology_, topology::TopologyBase::Create(settings, resolver_)); + std::atomic_store(&topology_, topology::TopologyBase::Create(settings, resolver_, blocking_task_processor_)); } ResultSet ClusterImpl::Execute( @@ -37,9 +40,10 @@ ResultSet ClusterImpl::Execute( const Query& query, const impl::ParameterList& parameters ) { + const auto resolved = ResolveCommandControl(command_control); return ExecuteImpl( - GetExecuteDeadline(ResolveNetworkTimeout(command_control)), - GetExecuteDeadline(ResolveStatementTimeout(command_control)), + GetExecuteDeadline(resolved.network_timeout.value_or(kDefaultStatementTimeout)), + resolved.statement_timeout.value_or(kDefaultStatementTimeout), flags, query, parameters @@ -48,7 +52,7 @@ ResultSet ClusterImpl::Execute( ResultSet ClusterImpl::ExecuteImpl( engine::Deadline acquire_deadline, - engine::Deadline statement_deadline, + std::chrono::milliseconds statement_timeout, ClusterHostTypeFlags flags, const Query& query, const impl::ParameterList& parameters @@ -61,11 +65,12 @@ ResultSet ClusterImpl::ExecuteImpl( auto conn = pool.Acquire(acquire_deadline); pool.AccountOutOfTransaction(); + const auto statement_deadline = GetExecuteDeadline(statement_timeout); CheckDeadlineNotExpired(statement_deadline); const auto start = utils::datetime::SteadyCoarseClock::now(); try { - auto result = conn->Query(query.GetStatementView(), parameters, std::min(acquire_deadline, statement_deadline)); + auto result = conn->Query(query, parameters, std::min(acquire_deadline, statement_deadline)); const auto elapsed = std::chrono::duration_cast< std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start); pool.AccountQueryExecuted(elapsed); @@ -82,35 +87,30 @@ ResultSet ClusterImpl::ExecuteImpl( Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags) { return Begin(flags, std::nullopt); } Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control) { - const auto acquire_deadline = GetExecuteDeadline(ResolveNetworkTimeout(command_control)); + const auto resolved = ResolveCommandControl(command_control); + const auto network_timeout = resolved.network_timeout.value_or(kDefaultStatementTimeout); + const auto statement_timeout = resolved.statement_timeout.value_or(kDefaultStatementTimeout); + const auto acquire_deadline = GetExecuteDeadline(network_timeout); CheckDeadlineNotExpired(acquire_deadline); tracing::Span span{storages::odbc::impl::tracing::kTransactionSpan}; const auto topology = std::atomic_load(&topology_); auto& pool = SelectPool(*topology, flags); auto connection = pool.Acquire(acquire_deadline); - const auto statement_deadline = GetExecuteDeadline(ResolveStatementTimeout(command_control)); - return Transaction{std::move(connection), pool, std::min(acquire_deadline, statement_deadline)}; + return Transaction{std::move(connection), pool, network_timeout, statement_timeout}; } -std::chrono::milliseconds ClusterImpl::ResolveNetworkTimeout(OptionalCommandControl command_control) const { - if (command_control && command_control->network_timeout) { - return *command_control->network_timeout; - } - if (const auto configured = GetDefaultNetworkTimeout()) { - return *configured; - } - return kDefaultStatementTimeout; -} - -std::chrono::milliseconds ClusterImpl::ResolveStatementTimeout(OptionalCommandControl command_control) const { - if (command_control && command_control->statement_timeout) { - return *command_control->statement_timeout; - } - if (const auto configured = GetDefaultStatementTimeout()) { - return *configured; +CommandControl ClusterImpl::ResolveCommandControl(OptionalCommandControl command_control) const { + auto resolved = default_command_control_.ReadCopy(); + if (command_control) { + if (command_control->network_timeout) { + resolved.network_timeout = command_control->network_timeout; + } + if (command_control->statement_timeout) { + resolved.statement_timeout = command_control->statement_timeout; + } } - return kDefaultStatementTimeout; + return resolved; } Pool& ClusterImpl::SelectPool(const topology::TopologyBase& topology, ClusterHostTypeFlags flags) { @@ -148,7 +148,7 @@ bool ClusterImpl::UpdateSettingsLocked(const settings::ODBCClusterSettings& sett // Construct and initialize all new pools before publishing. If this throws, // the currently working topology remains untouched. - auto new_topology = topology::TopologyBase::Create(settings, resolver_); + auto new_topology = topology::TopologyBase::Create(settings, resolver_, blocking_task_processor_); auto new_settings = std::make_shared(settings); settings_ = std::move(new_settings); std::atomic_store(&topology_, std::move(new_topology)); @@ -210,34 +210,14 @@ settings::ODBCClusterSettings ClusterImpl::MakeEffectiveSettingsLocked() const { return effective; } -void ClusterImpl::SetDefaultCommandControl(const CommandControl& cc) { - if (cc.network_timeout.has_value()) { - default_network_timeout_ms_.store(*cc.network_timeout); - has_network_timeout_.store(true, std::memory_order_release); - } else { - has_network_timeout_.store(false, std::memory_order_release); - } - - if (cc.statement_timeout.has_value()) { - default_statement_timeout_ms_.store(*cc.statement_timeout); - has_statement_timeout_.store(true, std::memory_order_release); - } else { - has_statement_timeout_.store(false, std::memory_order_release); - } -} +void ClusterImpl::SetDefaultCommandControl(const CommandControl& cc) { default_command_control_.Assign(cc); } std::optional ClusterImpl::GetDefaultNetworkTimeout() const { - if (has_network_timeout_.load(std::memory_order_acquire)) { - return std::chrono::milliseconds{default_network_timeout_ms_.load()}; - } - return std::nullopt; + return default_command_control_.ReadCopy().network_timeout; } std::optional ClusterImpl::GetDefaultStatementTimeout() const { - if (has_statement_timeout_.load(std::memory_order_acquire)) { - return std::chrono::milliseconds{default_statement_timeout_ms_.load()}; - } - return std::nullopt; + return default_command_control_.ReadCopy().statement_timeout; } } // namespace storages::odbc::detail diff --git a/odbc/src/storages/odbc/detail/cluster_impl.hpp b/odbc/src/storages/odbc/detail/cluster_impl.hpp index 93f72b328cdf..7de643b81aec 100644 --- a/odbc/src/storages/odbc/detail/cluster_impl.hpp +++ b/odbc/src/storages/odbc/detail/cluster_impl.hpp @@ -1,6 +1,5 @@ #pragma once -#include #include #include #include @@ -10,6 +9,7 @@ #include #include #include +#include #include #include @@ -30,7 +30,11 @@ namespace storages::odbc::detail { class ClusterImpl { public: - ClusterImpl(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver); + ClusterImpl( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor + ); ~ClusterImpl() = default; @@ -62,29 +66,26 @@ class ClusterImpl { ResultSet ExecuteImpl( engine::Deadline acquire_deadline, - engine::Deadline statement_deadline, + std::chrono::milliseconds statement_timeout, ClusterHostTypeFlags flags, const Query& query, const impl::ParameterList& parameters ); - std::chrono::milliseconds ResolveNetworkTimeout(OptionalCommandControl command_control) const; - std::chrono::milliseconds ResolveStatementTimeout(OptionalCommandControl command_control) const; + CommandControl ResolveCommandControl(OptionalCommandControl command_control) const; bool UpdateSettingsLocked(const settings::ODBCClusterSettings& settings); settings::ODBCClusterSettings MakeEffectiveSettingsLocked() const; clients::dns::Resolver* resolver_; + engine::TaskProcessor& blocking_task_processor_; std::shared_ptr topology_; mutable engine::Mutex settings_mutex_; std::shared_ptr settings_; std::shared_ptr baseline_settings_; std::optional pool_settings_override_; - // Dynamic config: command control (timeouts) - std::atomic default_network_timeout_ms_{std::chrono::milliseconds::zero()}; - std::atomic default_statement_timeout_ms_{std::chrono::milliseconds::zero()}; - std::atomic has_network_timeout_{false}; - std::atomic has_statement_timeout_{false}; + // One RCU value prevents readers from observing a torn dynamic-config update. + rcu::Variable default_command_control_; }; } // namespace storages::odbc::detail diff --git a/odbc/src/storages/odbc/detail/connection.cpp b/odbc/src/storages/odbc/detail/connection.cpp index 31802ba00835..eabc9ed446f9 100644 --- a/odbc/src/storages/odbc/detail/connection.cpp +++ b/odbc/src/storages/odbc/detail/connection.cpp @@ -1,10 +1,17 @@ #include #include +#include #include #include +#include +#include #include +#include +#include #include +#include +#include #include #include @@ -14,6 +21,11 @@ #include #include +#include +#include +#include +#include + #include #include #include @@ -27,6 +39,65 @@ namespace storages::odbc { namespace { +using StatementHandle = std::unique_ptr, void (*)(SQLHSTMT)>; + +template +auto RunBlocking(engine::TaskProcessor& task_processor, Func&& func) -> std::invoke_result_t { + const engine::TaskCancellationBlocker cancellation_blocker; + auto task = engine::CriticalAsyncNoTracing(task_processor, std::forward(func)); + return task.Get(); +} + +void CheckOperationInterrupted(engine::Deadline deadline) { + detail::CheckDeadlineNotExpired(deadline); + if (engine::current_task::ShouldCancel()) { + throw OperationInterrupted("Cancelled by task cancellation"); + } +} + +template +auto RunBlockingChecked(engine::TaskProcessor& task_processor, engine::Deadline deadline, Func&& func) + -> std::invoke_result_t { + CheckOperationInterrupted(deadline); + try { + if constexpr (std::is_void_v>) { + RunBlocking(task_processor, std::forward(func)); + CheckOperationInterrupted(deadline); + } else { + auto result = RunBlocking(task_processor, std::forward(func)); + CheckOperationInterrupted(deadline); + return result; + } + } catch (...) { + // Run the deadline check in the caller task so inherited-deadline + // accounting is applied to the request, not to the blocking worker. + CheckOperationInterrupted(deadline); + throw; + } +} + +void ConfigureDriverManager() { + static std::once_flag flag; + std::call_once(flag, [] { + const auto result = + SQLSetEnvAttr(SQL_NULL_HANDLE, SQL_ATTR_CONNECTION_POOLING, reinterpret_cast(SQL_CP_OFF), 0); + if (!SQL_SUCCEEDED(result)) { + throw ConnectionError("Failed to disable ODBC driver-manager connection pooling"); + } + }); +} + +template +Exception MakeDriverError(std::string message, SQLRETURN result, SQLHANDLE handle, SQLSMALLINT handle_type) { + auto diagnostics = detail::GetSQLDiagnostics(handle, handle_type); + const auto formatted = detail::FormatSQLDiagnostics(diagnostics); + if (!formatted.empty()) { + message += ": "; + message += formatted; + } + return Exception{std::move(message), std::move(diagnostics), result == SQL_INVALID_HANDLE}; +} + void DestroyEnvironmentHandle(SQLHENV handle) { if (handle != SQL_NULL_HENV) { SQLFreeHandle(SQL_HANDLE_ENV, handle); @@ -40,13 +111,41 @@ void DestroyDatabaseHandle(SQLHDBC handle) { } } +void DestroyStatementHandle(SQLHSTMT handle) { + if (handle != SQL_NULL_HSTMT) { + SQLFreeHandle(SQL_HANDLE_STMT, handle); + } +} + +void DestroyConnectionHandlesOnBlockingTaskProcessor( + engine::TaskProcessor& task_processor, + Connection::EnvironmentHandle& environment, + Connection::DatabaseHandle& database +) noexcept { + // Release before scheduling so a task-scheduling failure cannot invoke + // synchronous ODBC cleanup on the caller coroutine task processor. + const auto database_handle = database.release(); + const auto environment_handle = environment.release(); + try { + RunBlocking(task_processor, [database_handle, environment_handle] { + DestroyDatabaseHandle(database_handle); + DestroyEnvironmentHandle(environment_handle); + }); + } catch (const std::exception& ex) { + // At this point blocking in the caller is worse than leaking two + // handles while task processors are already unable to accept cleanup. + LOG_ERROR() << "Failed to schedule ODBC connection cleanup on blocking task processor: " << ex; + } catch (...) { + LOG_ERROR() << "Failed to schedule ODBC connection cleanup on blocking task processor"; + } +} + Connection::EnvironmentHandle MakeEnvironmentHandle() { SQLHENV env = SQL_NULL_HENV; SQLRETURN ret = SQLAllocHandle(SQL_HANDLE_ENV, SQL_NULL_HANDLE, &env); if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to allocate environment handle:" + detail::GetSQLDiagString(SQL_NULL_HANDLE, SQL_HANDLE_ENV) - ); + throw MakeDriverError< + ConnectionError>("Failed to allocate environment handle", ret, SQL_NULL_HANDLE, SQL_HANDLE_ENV); } return Connection::EnvironmentHandle(env, &DestroyEnvironmentHandle); @@ -56,14 +155,147 @@ Connection::DatabaseHandle MakeDatabaseHandle(SQLHENV env) { SQLHDBC dbc = SQL_NULL_HDBC; SQLRETURN ret = SQLAllocHandle(SQL_HANDLE_DBC, env, &dbc); if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to allocate connection handle:" + detail::GetSQLDiagString(SQL_NULL_HANDLE, SQL_HANDLE_DBC) - ); + throw MakeDriverError("Failed to allocate connection handle", ret, env, SQL_HANDLE_ENV); } return Connection::DatabaseHandle(dbc, &DestroyDatabaseHandle); } +StatementHandle MakeStatementHandle(SQLHDBC connection) { + SQLHSTMT statement = SQL_NULL_HSTMT; + const auto result = SQLAllocHandle(SQL_HANDLE_STMT, connection, &statement); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError< + StatementError>("Failed to allocate ODBC statement handle", result, connection, SQL_HANDLE_DBC); + } + return StatementHandle{statement, &DestroyStatementHandle}; +} + +void CheckStatementResult(SQLRETURN result, SQLHSTMT statement, std::string_view operation) { + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError< + StatementError>(fmt::format("Failed to {} ODBC statement", operation), result, statement, SQL_HANDLE_STMT); + } +} + +detail::ResultWrapper::Column DescribeColumn(SQLHSTMT statement, SQLUSMALLINT column) { + std::array buffer{}; + SQLSMALLINT name_length = 0; + SQLSMALLINT type = SQL_UNKNOWN_TYPE; + auto result = SQLDescribeCol( + statement, + column, + buffer.data(), + static_cast(buffer.size()), + &name_length, + &type, + nullptr, + nullptr, + nullptr + ); + CheckStatementResult(result, statement, "describe result column"); + + if (name_length >= static_cast(buffer.size())) { + std::vector long_buffer(static_cast(name_length) + 1); + result = SQLDescribeCol( + statement, + column, + long_buffer.data(), + static_cast(long_buffer.size()), + &name_length, + &type, + nullptr, + nullptr, + nullptr + ); + CheckStatementResult(result, statement, "describe result column"); + return { + std::string{reinterpret_cast(long_buffer.data()), static_cast(name_length)}, + type, + }; + } + + return { + std::string{reinterpret_cast(buffer.data()), static_cast(name_length)}, + type, + }; +} + +detail::ResultWrapper::Cell ReadCell(SQLHSTMT statement, SQLUSMALLINT column, engine::Deadline deadline) { + constexpr std::size_t kChunkSize = 4096; + std::array buffer{}; + std::string value; + + while (true) { + SQLLEN indicator = 0; + detail::CheckDeadlineNotExpired(deadline); + const auto result = + SQLGetData(statement, column, SQL_C_CHAR, buffer.data(), static_cast(buffer.size()), &indicator); + detail::CheckDeadlineNotExpired(deadline); + + if (result == SQL_NO_DATA) { + break; + } + CheckStatementResult(result, statement, "read result column"); + if (indicator == SQL_NULL_DATA) { + return {std::nullopt}; + } + + const auto terminator = std::find(buffer.begin(), buffer.end(), static_cast('\0')); + const auto chunk_size = static_cast(terminator - buffer.begin()); + value.append(reinterpret_cast(buffer.data()), chunk_size); + + if (result == SQL_SUCCESS) { + break; + } + if (chunk_size == 0) { + throw ResultSetError("ODBC driver returned SQL_SUCCESS_WITH_INFO without result data progress"); + } + } + + return {std::move(value)}; +} + +std::shared_ptr MaterializeResult(SQLHSTMT statement, engine::Deadline deadline) { + SQLSMALLINT column_count = 0; + CheckStatementResult(SQLNumResultCols(statement, &column_count), statement, "get result column count for"); + + std::size_t rows_affected = 0; + if (column_count == 0) { + SQLLEN affected = 0; + const auto row_count_result = SQLRowCount(statement, &affected); + rows_affected = SQL_SUCCEEDED(row_count_result) && affected > 0 ? static_cast(affected) : 0; + } + + std::vector columns; + columns.reserve(static_cast(column_count)); + for (SQLSMALLINT index = 0; index < column_count; ++index) { + columns.push_back(DescribeColumn(statement, static_cast(index + 1))); + } + + std::vector rows; + if (column_count > 0) { + while (true) { + detail::CheckDeadlineNotExpired(deadline); + const auto fetch_result = SQLFetch(statement); + detail::CheckDeadlineNotExpired(deadline); + if (fetch_result == SQL_NO_DATA) { + break; + } + CheckStatementResult(fetch_result, statement, "fetch row from"); + + detail::ResultWrapper::Row row; + row.reserve(static_cast(column_count)); + for (SQLSMALLINT index = 0; index < column_count; ++index) { + row.push_back(ReadCell(statement, static_cast(index + 1), deadline)); + } + rows.push_back(std::move(row)); + } + } + + return std::make_shared(std::move(columns), std::move(rows), rows_affected); +} + struct ParameterBinding final { SQLSMALLINT c_type; SQLSMALLINT sql_type; @@ -73,7 +305,7 @@ struct ParameterBinding final { }; struct BoundParameter final { - using Value = std::variant; + using Value = std::variant; explicit BoundParameter(const impl::Parameter& parameter) : type{parameter.GetType()}, @@ -88,8 +320,13 @@ struct BoundParameter final { return static_cast(parameter.Get() ? 1 : 0); case ParameterType::kSignedInteger: return static_cast(parameter.Get()); - case ParameterType::kUnsignedInteger: - return static_cast(parameter.Get()); + case ParameterType::kUnsignedInteger: { + const auto value = parameter.Get(); + if (value > static_cast(std::numeric_limits::max())) { + throw StatementError("ODBC unsigned integer parameter is outside the portable SQL BIGINT range"); + } + return static_cast(value); + } case ParameterType::kFloatingPoint: return static_cast(parameter.Get()); case ParameterType::kString: @@ -126,11 +363,11 @@ ParameterBinding GetParameterBinding(BoundParameter& parameter) { }; case ParameterType::kUnsignedInteger: return { - SQL_C_UBIGINT, + SQL_C_SBIGINT, SQL_BIGINT, - 20, - &std::get(parameter.value), - static_cast(sizeof(SQLUBIGINT)), + 19, + &std::get(parameter.value), + static_cast(sizeof(SQLBIGINT)), }; case ParameterType::kFloatingPoint: return { @@ -168,9 +405,8 @@ void BindParameters(SQLHSTMT statement, const impl::ParameterList& parameters, e SQLSMALLINT expected_count = 0; const auto count_result = SQLNumParams(statement, &expected_count); if (!SQL_SUCCEEDED(count_result)) { - throw StatementError( - "Failed to determine ODBC parameter count:" + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) - ); + throw MakeDriverError< + StatementError>("Failed to determine ODBC parameter count", count_result, statement, SQL_HANDLE_STMT); } if (static_cast(expected_count) != parameters.size()) { throw StatementError( @@ -201,10 +437,12 @@ void BindParameters(SQLHSTMT statement, const impl::ParameterList& parameters, e &nullable ); if (!SQL_SUCCEEDED(describe_result)) { - throw StatementError( + throw MakeDriverError( "Cannot infer the type of a NULL ODBC parameter; add an explicit SQL cast or use a typed " - "std::optional where supported by the driver:" + - detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) + "std::optional where supported by the driver", + describe_result, + statement, + SQL_HANDLE_STMT ); } } @@ -223,9 +461,11 @@ void BindParameters(SQLHSTMT statement, const impl::ParameterList& parameters, e &indicators[index] ); if (!SQL_SUCCEEDED(bind_result)) { - throw StatementError( - fmt::format("Failed to bind ODBC parameter {}:", index + 1) + - detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) + throw MakeDriverError( + fmt::format("Failed to bind ODBC parameter {}", index + 1), + bind_result, + statement, + SQL_HANDLE_STMT ); } } @@ -234,9 +474,8 @@ void BindParameters(SQLHSTMT statement, const impl::ParameterList& parameters, e const auto execute_result = SQLExecute(statement); detail::CheckDeadlineNotExpired(deadline); if (!SQL_SUCCEEDED(execute_result) && execute_result != SQL_NO_DATA) { - throw StatementError( - "Failed to execute prepared ODBC query:" + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT) - ); + throw MakeDriverError< + StatementError>("Failed to execute prepared ODBC query", execute_result, statement, SQL_HANDLE_STMT); } } @@ -260,7 +499,8 @@ SQLRETURN ExecuteStatement( const auto prepare_result = SQLPrepare(statement, query_buffer.data(), SQL_NTS); detail::CheckDeadlineNotExpired(deadline); if (!SQL_SUCCEEDED(prepare_result)) { - throw StatementError("Failed to prepare ODBC query:" + detail::GetSQLDiagString(statement, SQL_HANDLE_STMT)); + throw MakeDriverError< + StatementError>("Failed to prepare ODBC query", prepare_result, statement, SQL_HANDLE_STMT); } BindParameters(statement, parameters, deadline); return SQL_SUCCESS; @@ -269,74 +509,84 @@ SQLRETURN ExecuteStatement( } // namespace Connection::Connection(const std::string& dsn) - : Connection{dsn, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)} + : Connection{dsn, engine::current_task::GetBlockingTaskProcessor(), detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)} {} Connection::Connection(const std::string& dsn, engine::Deadline deadline) - : env_(MakeEnvironmentHandle()), - handle_(Connection::DatabaseHandle(SQL_NULL_HDBC, &DestroyDatabaseHandle)) -{ - detail::CheckDeadlineNotExpired(deadline); - SQLRETURN ret = - SQLSetEnvAttr(env_.get(), SQL_ATTR_CONNECTION_POOLING, reinterpret_cast(SQL_CP_ONE_PER_DRIVER), 0); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to set connection pooling attribute:" + detail::GetSQLDiagString(env_.get(), SQL_HANDLE_ENV) - ); - } - - ret = SQLSetEnvAttr(env_.get(), SQL_ATTR_ODBC_VERSION, reinterpret_cast(SQL_OV_ODBC3), 0); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError("Failed to set ODBC version:" + detail::GetSQLDiagString(env_.get(), SQL_HANDLE_ENV)); - } - - handle_ = MakeDatabaseHandle(env_.get()); + : Connection{dsn, engine::current_task::GetBlockingTaskProcessor(), deadline} +{} - if (deadline.IsReachable()) { - const auto time_left = deadline.TimeLeft(); - if (time_left <= engine::Deadline::Duration::zero()) { - detail::CheckDeadlineNotExpired(deadline); +Connection::Connection( + const std::string& dsn, + engine::TaskProcessor& blocking_task_processor, + engine::Deadline deadline +) + : blocking_task_processor_{blocking_task_processor}, + env_{SQL_NULL_HENV, &DestroyEnvironmentHandle}, + handle_{SQL_NULL_HDBC, &DestroyDatabaseHandle} +{ + CheckOperationInterrupted(deadline); + auto handles = RunBlocking(blocking_task_processor_, [dsn, deadline] { + ConfigureDriverManager(); + auto environment = MakeEnvironmentHandle(); + auto result = + SQLSetEnvAttr(environment.get(), SQL_ATTR_ODBC_VERSION, reinterpret_cast(SQL_OV_ODBC3), 0); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError< + ConnectionError>("Failed to set ODBC version", result, environment.get(), SQL_HANDLE_ENV); } - const auto timeout = std::chrono::ceil(time_left); - const auto timeout_seconds = static_cast(timeout.count()); - ret = SQLSetConnectAttr( - handle_.get(), - SQL_ATTR_LOGIN_TIMEOUT, - reinterpret_cast(static_cast(timeout_seconds)), - SQL_IS_UINTEGER - ); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to set ODBC login timeout:" + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) + + auto database = MakeDatabaseHandle(environment.get()); + if (deadline.IsReachable()) { + const auto time_left = deadline.TimeLeft(); + if (time_left <= engine::Deadline::Duration::zero()) { + throw OperationInterrupted("Cancelled by deadline"); + } + const auto timeout = std::chrono::ceil(time_left); + const auto timeout_seconds = static_cast(timeout.count()); + result = SQLSetConnectAttr( + database.get(), + SQL_ATTR_LOGIN_TIMEOUT, + reinterpret_cast(static_cast(timeout_seconds)), + SQL_IS_UINTEGER ); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError< + ConnectionError>("Failed to set ODBC login timeout", result, database.get(), SQL_HANDLE_DBC); + } } - } - std::vector dsn_buffer(dsn.begin(), dsn.end()); - dsn_buffer.push_back('\0'); - detail::CheckDeadlineNotExpired(deadline); - ret = - SQLDriverConnect(handle_.get(), nullptr, dsn_buffer.data(), SQL_NTS, nullptr, 0, nullptr, SQL_DRIVER_COMPLETE); - detail::CheckDeadlineNotExpired(deadline); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to connect to database: " + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); - } - SQLUINTEGER scroll_option = 0; - ret = SQLGetInfo(handle_.get(), SQL_SCROLL_OPTIONS, &scroll_option, sizeof(scroll_option), nullptr); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to get scroll options:" + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) + std::vector dsn_buffer(dsn.begin(), dsn.end()); + dsn_buffer.push_back('\0'); + result = SQLDriverConnect( + database.get(), + nullptr, + dsn_buffer.data(), + SQL_NTS, + nullptr, + 0, + nullptr, + SQL_DRIVER_NOPROMPT ); - } + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError< + ConnectionError>("Failed to connect to database", result, database.get(), SQL_HANDLE_DBC); + } + return std::pair{std::move(environment), std::move(database)}; + }); - // TODO: add support for other scroll options - if (!(scroll_option & SQL_FD_FETCH_ABSOLUTE)) { - throw ConnectionError("SQL_FD_FETCH_ABSOLUTE is not supported"); + try { + CheckOperationInterrupted(deadline); + } catch (...) { + DestroyConnectionHandlesOnBlockingTaskProcessor(blocking_task_processor_, handles.first, handles.second); + throw; } + env_ = std::move(handles.first); + handle_ = std::move(handles.second); } +Connection::~Connection() { DestroyConnectionHandlesOnBlockingTaskProcessor(blocking_task_processor_, env_, handle_); } + ResultSet Connection::Query(std::string_view query) { return Query(query, impl::ParameterList{}, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)); } @@ -350,146 +600,197 @@ ResultSet Connection::Query(std::string_view query, const impl::ParameterList& p } ResultSet Connection::Query(std::string_view query, const impl::ParameterList& parameters, engine::Deadline deadline) { - detail::CheckDeadlineNotExpired(deadline); - - auto guard = GetBrokenGuard(); - return guard.Execute([&] { - tracing::Span span{detail::tracing::MakeQuerySpanName(query)}; - span.AddTag(tracing::kDatabaseType, "odbc"); - span.AddTag(tracing::kDatabaseStatement, std::string{query}); + return Query(storages::odbc::Query{std::string{query}}, parameters, deadline); +} - auto stmt = detail::MakeResultHandle(handle_.get()); +ResultSet Connection::Query( + const storages::odbc::Query& query, + const impl::ParameterList& parameters, + engine::Deadline deadline +) { + CheckOperationInterrupted(deadline); + + const auto statement = query.GetStatementView(); + tracing::Span span{detail::tracing::MakeQuerySpanName(statement)}; + span.AddTag(tracing::kDatabaseType, "odbc"); + const auto span_tags = detail::tracing::MakeQuerySpanTags(query); + if (span_tags.statement_name) { + span.AddTag(tracing::kDatabaseStatementName, std::string{*span_tags.statement_name}); + } + if (span_tags.statement) { + span.AddTag(tracing::kDatabaseStatement, std::string{*span_tags.statement}); + } - if (deadline.IsReachable()) { - const auto left = deadline.TimeLeft(); - if (left <= engine::Deadline::Duration::zero()) { - detail::CheckDeadlineNotExpired(deadline); - } - const auto seconds = std::chrono::ceil(left); - const auto timeout_sec = static_cast(seconds.count()); - /* ODBC SQL_ATTR_QUERY_TIMEOUT is in whole seconds; deadline checks still use full TimeLeft() - * resolution. */ - const auto timeout_result = SQLSetStmtAttr( - stmt.get(), - SQL_ATTR_QUERY_TIMEOUT, - reinterpret_cast(static_cast(timeout_sec)), - 0 + auto guard = GetBrokenGuard(); + try { + return guard.Execute([&] { + return RunBlockingChecked( + blocking_task_processor_, + deadline, + [this, query = std::string{statement}, parameters, deadline] { + const std::lock_guard lock{handle_mutex_}; + try { + auto stmt = MakeStatementHandle(handle_.get()); + + if (deadline.IsReachable()) { + const auto left = deadline.TimeLeft(); + if (left <= engine::Deadline::Duration::zero()) { + throw OperationInterrupted("Cancelled by deadline"); + } + const auto seconds = std::chrono::ceil(left); + const auto timeout_sec = static_cast(seconds.count()); + // SQL_ATTR_QUERY_TIMEOUT is in whole seconds; the + // exact deadline is checked in the caller afterwards. + const auto timeout_result = SQLSetStmtAttr( + stmt.get(), + SQL_ATTR_QUERY_TIMEOUT, + reinterpret_cast(static_cast(timeout_sec)), + 0 + ); + if (!SQL_SUCCEEDED(timeout_result)) { + throw MakeDriverError( + "Failed to set ODBC query timeout", + timeout_result, + stmt.get(), + SQL_HANDLE_STMT + ); + } + } + + const auto result = ExecuteStatement(stmt.get(), query, parameters, deadline); + if (!SQL_SUCCEEDED(result) && result != SQL_NO_DATA) { + throw MakeDriverError< + StatementError>("Failed to execute query", result, stmt.get(), SQL_HANDLE_STMT); + } + return ResultSet(MaterializeResult(stmt.get(), deadline)); + } catch (const StatementError& ex) { + if (ex.IsInvalidHandle() || detail::HasConnectionError(ex.GetDiagnostics())) { + NotifyBroken(); + } else { + UpdateBrokenFromDriver(); + } + throw; + } + } ); - if (!SQL_SUCCEEDED(timeout_result)) { - throw StatementError( - "Failed to set ODBC query timeout:" + detail::GetSQLDiagString(stmt.get(), SQL_HANDLE_STMT) - ); - } - } + }); + } catch (const OperationInterrupted&) { + // The state after a synchronous call that crossed a deadline or was + // cancelled is uncertain. Never return this HDBC to the pool. + NotifyBroken(); + throw; + } catch (const Error& ex) { + span.AddTag(tracing::kErrorFlag, true); + span.AddTag(tracing::kErrorMessage, ex.what()); + throw; + } +} - SQLRETURN ret = ExecuteStatement(stmt.get(), query, parameters, deadline); - if (!SQL_SUCCEEDED(ret) && ret != SQL_NO_DATA) { - const auto diag = detail::GetSQLDiagString(stmt.get(), SQL_HANDLE_STMT); - span.AddTag(tracing::kErrorFlag, true); - span.AddTag(tracing::kErrorMessage, diag); - throw StatementError("Failed to execute query:" + diag); - } +bool Connection::IsBroken() const { return broken_.load() || in_transaction_.load(); } - // Only call Fetch for SELECT-like statements that produce a result set. - // DML statements (INSERT/UPDATE/DELETE) have 0 result columns; calling - // SQLFetch on them returns SQL_NO_DATA or an error depending on the driver. - if (ret != SQL_NO_DATA) { - SQLSMALLINT col_count = 0; - SQLNumResultCols(stmt.get(), &col_count); - if (col_count > 0) { - auto wrapper = std::make_shared(std::move(stmt)); - wrapper->Fetch(); - return ResultSet(std::move(wrapper)); - } - } +bool Connection::IsMarkedBroken() const noexcept { return broken_.load(); } - return ResultSet(std::make_shared(std::move(stmt))); - }); -} +void Connection::NotifyBroken() { broken_.store(true); } -bool Connection::DriverReportsDead() const { - SQLUINTEGER state = 0; - SQLRETURN ret = SQLGetConnectAttr(handle_.get(), SQL_ATTR_CONNECTION_DEAD, &state, sizeof(state), nullptr); - if (!SQL_SUCCEEDED(ret) || state == SQL_CD_TRUE) { - return true; +void Connection::UpdateBrokenFromDriver() noexcept { + SQLUINTEGER state = SQL_CD_TRUE; + const auto result = SQLGetConnectAttr(handle_.get(), SQL_ATTR_CONNECTION_DEAD, &state, sizeof(state), nullptr); + if (!SQL_SUCCEEDED(result) || state == SQL_CD_TRUE) { + NotifyBroken(); } - - return false; } -bool Connection::IsBroken() const { return broken_.load() || DriverReportsDead(); } - -void Connection::NotifyBroken() { broken_.store(true); } - detail::BrokenGuard Connection::GetBrokenGuard() { return detail::BrokenGuard{*this}; } -bool Connection::IsInsideTransaction() const { - SQLUINTEGER state = 0; - SQLRETURN ret = SQLGetConnectAttr(handle_.get(), SQL_ATTR_AUTOCOMMIT, &state, sizeof(state), nullptr); - if (!SQL_SUCCEEDED(ret) || state == SQL_AUTOCOMMIT_OFF) { - return true; - } - return false; -} +bool Connection::IsInsideTransaction() const noexcept { return in_transaction_.load(); } void Connection::Begin(engine::Deadline deadline) { auto guard = GetBrokenGuard(); - guard.Execute([this, deadline] { - detail::CheckDeadlineNotExpired(deadline); - SQLRETURN ret = SQLSetConnectAttr( - handle_.get(), - SQL_ATTR_AUTOCOMMIT, - reinterpret_cast(SQL_AUTOCOMMIT_OFF), - SQL_IS_UINTEGER - ); - - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to set connection autocommit attribute:" + - detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); - } - }); + try { + guard.Execute([this, deadline] { + RunBlockingChecked(blocking_task_processor_, deadline, [this, deadline] { + const std::lock_guard lock{handle_mutex_}; + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLSetConnectAttr( + handle_.get(), + SQL_ATTR_AUTOCOMMIT, + reinterpret_cast(SQL_AUTOCOMMIT_OFF), + SQL_IS_UINTEGER + ); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError( + "Failed to set connection autocommit attribute", + result, + handle_.get(), + SQL_HANDLE_DBC + ); + } + in_transaction_.store(true); + }); + }); + } catch (const OperationInterrupted&) { + // SQLSetConnectAttr may have completed before the caller observed the + // deadline/cancellation. Do not pool a connection with uncertain + // autocommit state. + NotifyBroken(); + throw; + } } void Connection::Commit(engine::Deadline deadline) { auto guard = GetBrokenGuard(); - guard.Execute([this, deadline] { - detail::CheckDeadlineNotExpired(deadline); - if (!IsInsideTransaction()) { - throw ConnectionError( - "User try to commit autocommit connection:" + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); - } - SQLRETURN ret = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), SQL_COMMIT); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to commit transaction inside connection:" + - detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); - } - RestoreAutocommit(); - }); + try { + guard.Execute([this, deadline] { + if (!IsInsideTransaction()) { + throw ConnectionError("Cannot commit an ODBC connection outside a transaction"); + } + RunBlockingChecked(blocking_task_processor_, deadline, [this, deadline] { + const std::lock_guard lock{handle_mutex_}; + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), SQL_COMMIT); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError( + "Failed to commit transaction inside connection", + result, + handle_.get(), + SQL_HANDLE_DBC + ); + } + RestoreAutocommit(); + }); + }); + } catch (const OperationInterrupted&) { + NotifyBroken(); + throw; + } } void Connection::Rollback(engine::Deadline deadline) { auto guard = GetBrokenGuard(); - guard.Execute([this, deadline] { - detail::CheckDeadlineNotExpired(deadline); - if (!IsInsideTransaction()) { - throw ConnectionError( - "User try to rollback autocommit connection:" + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); - } - SQLRETURN ret = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), SQL_ROLLBACK); - if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to rollback transaction inside connection:" + - detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); - } - RestoreAutocommit(); - }); + try { + guard.Execute([this, deadline] { + if (!IsInsideTransaction()) { + throw ConnectionError("Cannot roll back an ODBC connection outside a transaction"); + } + RunBlockingChecked(blocking_task_processor_, deadline, [this, deadline] { + const std::lock_guard lock{handle_mutex_}; + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), SQL_ROLLBACK); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError( + "Failed to rollback transaction inside connection", + result, + handle_.get(), + SQL_HANDLE_DBC + ); + } + RestoreAutocommit(); + }); + }); + } catch (const OperationInterrupted&) { + NotifyBroken(); + throw; + } } void Connection::RestoreAutocommit() { @@ -500,10 +801,10 @@ void Connection::RestoreAutocommit() { SQL_IS_UINTEGER ); if (!SQL_SUCCEEDED(ret)) { - throw ConnectionError( - "Failed to restore autocommit after transaction:" + detail::GetSQLDiagString(handle_.get(), SQL_HANDLE_DBC) - ); + throw MakeDriverError< + ConnectionError>("Failed to restore autocommit after transaction", ret, handle_.get(), SQL_HANDLE_DBC); } + in_transaction_.store(false); } } // namespace storages::odbc diff --git a/odbc/src/storages/odbc/detail/connection.hpp b/odbc/src/storages/odbc/detail/connection.hpp index 255ac899b9a2..7f9a20115a35 100644 --- a/odbc/src/storages/odbc/detail/connection.hpp +++ b/odbc/src/storages/odbc/detail/connection.hpp @@ -1,16 +1,19 @@ #pragma once #include +#include #include #include #include +#include #include #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -29,8 +32,9 @@ class Connection final { explicit Connection(const std::string& dsn); Connection(const std::string& dsn, engine::Deadline deadline); + Connection(const std::string& dsn, engine::TaskProcessor& blocking_task_processor, engine::Deadline deadline); - ~Connection() = default; + ~Connection(); Connection(const Connection&) = delete; Connection& operator=(const Connection&) = delete; @@ -48,8 +52,15 @@ class Connection final { ResultSet Query(std::string_view query, const impl::ParameterList& parameters, engine::Deadline deadline); + ResultSet Query( + const storages::odbc::Query& query, + const impl::ParameterList& parameters, + engine::Deadline deadline + ); + // required by ConnectionPool bool IsBroken() const; + bool IsMarkedBroken() const noexcept; void NotifyBroken(); detail::BrokenGuard GetBrokenGuard(); @@ -59,14 +70,16 @@ class Connection final { void Begin(engine::Deadline deadline); void Commit(engine::Deadline deadline); void Rollback(engine::Deadline deadline); - bool IsInsideTransaction() const; // check if connection has autocommit_off transaction mode + bool IsInsideTransaction() const noexcept; void RestoreAutocommit(); - bool DriverReportsDead() const; - + void UpdateBrokenFromDriver() noexcept; + engine::TaskProcessor& blocking_task_processor_; + mutable std::mutex handle_mutex_; EnvironmentHandle env_; DatabaseHandle handle_; std::atomic broken_{false}; + std::atomic in_transaction_{false}; }; } // namespace storages::odbc diff --git a/odbc/src/storages/odbc/detail/deadline.cpp b/odbc/src/storages/odbc/detail/deadline.cpp index d1fc0361fe5e..13df68b94241 100644 --- a/odbc/src/storages/odbc/detail/deadline.cpp +++ b/odbc/src/storages/odbc/detail/deadline.cpp @@ -19,7 +19,10 @@ void CheckDeadlineNotExpired(const engine::Deadline& deadline) { return; } if (deadline.IsReached()) { - server::request::MarkTaskInheritedDeadlineExpired(); + const auto inherited = server::request::GetTaskInheritedDeadline(); + if (inherited.IsReachable() && inherited.IsReached()) { + server::request::MarkTaskInheritedDeadlineExpired(); + } throw OperationInterrupted("Cancelled by deadline"); } } diff --git a/odbc/src/storages/odbc/detail/deadline.hpp b/odbc/src/storages/odbc/detail/deadline.hpp index e0900773761e..e97b768d0319 100644 --- a/odbc/src/storages/odbc/detail/deadline.hpp +++ b/odbc/src/storages/odbc/detail/deadline.hpp @@ -12,6 +12,9 @@ namespace storages::odbc::detail { /// Milliseconds so defaults and settings can be sub-second; pool wait / @ref engine::Deadline use full resolution. inline constexpr std::chrono::milliseconds kDefaultStatementTimeout{10000}; +/// Cleanup must remain possible after a request/statement deadline expires. +inline constexpr std::chrono::milliseconds kDefaultCleanupTimeout{5000}; + /// Combines task-inherited request deadline (if any) with \a operation_deadline, returning the /// earlier of the two (like storages::postgres::AdjustTimeout). engine::Deadline MergeWithInheritedDeadline(engine::Deadline operation_deadline) noexcept; diff --git a/odbc/src/storages/odbc/detail/diag_wrapper.cpp b/odbc/src/storages/odbc/detail/diag_wrapper.cpp index e04727e6e114..ca672c891299 100644 --- a/odbc/src/storages/odbc/detail/diag_wrapper.cpp +++ b/odbc/src/storages/odbc/detail/diag_wrapper.cpp @@ -1,30 +1,69 @@ #include #include +#include +#include + USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail { -std::string GetSQLDiagString(SQLHANDLE handle, SQLSMALLINT type) { - std::string result; +std::vector GetSQLDiagnostics(SQLHANDLE handle, SQLSMALLINT type) { + std::vector diagnostics; for (SQLINTEGER i = 1;; ++i) { SQLINTEGER native = 0; - SQLCHAR state[7]; - SQLCHAR text[SQL_MAX_MESSAGE_LENGTH]; + std::array state{}; + std::array text{}; SQLSMALLINT len = 0; - const auto ret = SQLGetDiagRec(type, handle, i, state, &native, text, sizeof(text), &len); + const auto ret = SQLGetDiagRec( + type, + handle, + static_cast(i), + state.data(), + &native, + text.data(), + static_cast(text.size()), + &len + ); if (!SQL_SUCCEEDED(ret)) { break; } - result += fmt::format("{} (code {})", reinterpret_cast(&text[0]), native); + const auto message_length = std::min(len > 0 ? static_cast(len) : 0, text.size() - 1); + diagnostics.push_back(DiagnosticRecord{ + .sql_state = std::string{reinterpret_cast(state.data()), SQL_SQLSTATE_SIZE}, + .native_error = static_cast(native), + .message = std::string{reinterpret_cast(text.data()), message_length}, + }); } + return diagnostics; +} + +std::string FormatSQLDiagnostics(const std::vector& diagnostics) { + std::string result; + for (const auto& diagnostic : diagnostics) { + if (!result.empty()) { + result += "; "; + } + result += + fmt::format("[{}] {} (native code {})", diagnostic.sql_state, diagnostic.message, diagnostic.native_error); + } return result; } +std::string GetSQLDiagString(SQLHANDLE handle, SQLSMALLINT type) { + return FormatSQLDiagnostics(GetSQLDiagnostics(handle, type)); +} + +bool HasConnectionError(const std::vector& diagnostics) noexcept { + return std::any_of(diagnostics.begin(), diagnostics.end(), [](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state.size() >= 2 && diagnostic.sql_state[0] == '0' && diagnostic.sql_state[1] == '8'; + }); +} + } // namespace storages::odbc::detail USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/detail/diag_wrapper.hpp b/odbc/src/storages/odbc/detail/diag_wrapper.hpp index 65207d1cde0d..c7e359709222 100644 --- a/odbc/src/storages/odbc/detail/diag_wrapper.hpp +++ b/odbc/src/storages/odbc/detail/diag_wrapper.hpp @@ -3,13 +3,23 @@ #include #include #include +#include +#include + +#include USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail { +std::vector GetSQLDiagnostics(SQLHANDLE handle, SQLSMALLINT type); + +std::string FormatSQLDiagnostics(const std::vector& diagnostics); + std::string GetSQLDiagString(SQLHANDLE handle, SQLSMALLINT type); +bool HasConnectionError(const std::vector& diagnostics) noexcept; + } // namespace storages::odbc::detail USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/detail/pool.cpp b/odbc/src/storages/odbc/detail/pool.cpp index cbeab96b3faa..5c7746ad3ac0 100644 --- a/odbc/src/storages/odbc/detail/pool.cpp +++ b/odbc/src/storages/odbc/detail/pool.cpp @@ -1,6 +1,8 @@ #include #include +#include +#include #include #include #include @@ -12,59 +14,85 @@ namespace storages::odbc::detail { namespace { -auto constexpr kInitTimeout = std::chrono::milliseconds{1000}; +auto constexpr kConnectionSetupTimeout = std::chrono::milliseconds{2000}; +auto constexpr kMaxSimultaneouslyConnectingClients = std::size_t{5}; +auto constexpr kPoolSizeMonitorInterval = std::chrono::milliseconds{2000}; } // namespace -Pool::Pool(const std::string& dsn, std::size_t min_pool_size, std::size_t max_pool_size) - : ConnectionPoolBase(max_pool_size, max_pool_size), +Pool::Pool( + const std::string& dsn, + std::size_t min_pool_size, + std::size_t max_pool_size, + engine::TaskProcessor& blocking_task_processor +) + : ConnectionPoolBase(max_pool_size, kMaxSimultaneouslyConnectingClients), dsns_({dsn}), - max_pool_size_(max_pool_size) + min_pool_size_(min_pool_size), + max_pool_size_(max_pool_size), + blocking_task_processor_{blocking_task_processor} { stats_.connection.maximum = max_pool_size; try { - Init(min_pool_size, kInitTimeout); + Init(min_pool_size_, kConnectionSetupTimeout); } catch (const std::exception& ex) { - LOG_ERROR() << "Error while initializing ODBC connection pool: " << ex; - Reset(); - throw; + // Temporary startup outages are recovered by the size monitor. Any + // connections that did initialize successfully remain available. + LOG_WARNING() << "ODBC pool initialized below min_pool_size: " << ex; } catch (...) { - Reset(); - throw; + LOG_WARNING() << "ODBC pool initialized below min_pool_size due to an unknown error"; } + size_monitor_.Start("odbc_connection_pool_monitor", {{kPoolSizeMonitorInterval}}, [this] { RunSizeMonitor(); }); } -Pool::Pool(std::vector dsns, std::size_t min_pool_size, std::size_t max_pool_size) - : ConnectionPoolBase(max_pool_size, max_pool_size), +Pool::Pool( + std::vector dsns, + std::size_t min_pool_size, + std::size_t max_pool_size, + engine::TaskProcessor& blocking_task_processor +) + : ConnectionPoolBase(max_pool_size, kMaxSimultaneouslyConnectingClients), dsns_(std::move(dsns)), - max_pool_size_(max_pool_size) + min_pool_size_(min_pool_size), + max_pool_size_(max_pool_size), + blocking_task_processor_{blocking_task_processor} { stats_.connection.maximum = max_pool_size; try { - Init(min_pool_size, kInitTimeout); + Init(min_pool_size_, kConnectionSetupTimeout); } catch (const std::exception& ex) { - LOG_ERROR() << "Error while initializing ODBC connection pool: " << ex; - Reset(); - throw; + LOG_WARNING() << "ODBC pool initialized below min_pool_size: " << ex; } catch (...) { - Reset(); - throw; + LOG_WARNING() << "ODBC pool initialized below min_pool_size due to an unknown error"; } + size_monitor_.Start("odbc_connection_pool_monitor", {{kPoolSizeMonitorInterval}}, [this] { RunSizeMonitor(); }); } -Pool::~Pool() { Reset(); } +Pool::~Pool() { + size_monitor_.Stop(); + Reset(); +} ConnectionPtr Pool::Acquire(engine::Deadline deadline) { const auto start = utils::datetime::SteadyCoarseClock::now(); ++stats_.connection.waiting; const utils::FastScopeGuard waiting_guard([this]() noexcept { --stats_.connection.waiting; }); - auto conn_wrapper = AcquireConnection(deadline); + decltype(AcquireConnection(deadline)) conn_wrapper; + try { + conn_wrapper = AcquireConnection(deadline); + } catch (const drivers::impl::PoolWaitLimitExceededError&) { + CheckDeadlineNotExpired(deadline); + if (engine::current_task::ShouldCancel()) { + throw OperationInterrupted("Cancelled while waiting for an ODBC connection"); + } + throw PoolError("ODBC connection pool wait limit exceeded"); + } ++stats_.connection.used; const auto elapsed = std::chrono::duration_cast< - std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start); + std::chrono::milliseconds>(utils::datetime::SteadyCoarseClock::now() - start); stats_.acquire_percentile.Account(elapsed.count()); return {std::move(conn_wrapper.pool_ptr), std::move(conn_wrapper.connection_ptr)}; @@ -80,11 +108,11 @@ Pool::ConnectionUniquePtr Pool::DoCreateConnection(engine::Deadline deadline) { try { CheckDeadlineNotExpired(deadline); const auto idx = dsn_index_.fetch_add(1); - auto conn = std::make_unique(dsns_[idx % dsns_.size()], deadline); + auto conn = std::make_unique(dsns_[idx % dsns_.size()], blocking_task_processor_, deadline); ++stats_.connection.open_total; const auto elapsed = std::chrono::duration_cast< - std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start); + std::chrono::milliseconds>(utils::datetime::SteadyCoarseClock::now() - start); stats_.connection_percentile.Account(elapsed.count()); return conn; @@ -99,6 +127,19 @@ Pool::ConnectionUniquePtr Pool::DoCreateConnection(engine::Deadline deadline) { } } +void Pool::RunSizeMonitor() { + if (AliveConnectionsCountApprox() >= min_pool_size_) { + return; + } + try { + PushConnection(engine::Deadline::FromDuration(kConnectionSetupTimeout)); + } catch (const std::exception& ex) { + LOG_WARNING() << "Failed to restore ODBC min_pool_size: " << ex; + } catch (...) { + LOG_WARNING() << "Failed to restore ODBC min_pool_size due to an unknown error"; + } +} + void Pool::AccountConnectionCreated() noexcept { ++stats_.connection.active; } void Pool::AccountConnectionAcquired() noexcept {} @@ -114,7 +155,7 @@ void Pool::AccountOverload() noexcept { ++stats_.pool_exhaust_errors; } void Pool::AccountQueryExecuted(std::chrono::microseconds duration) noexcept { ++stats_.transaction.execute_total; - stats_.transaction.busy_percentile.Account(duration.count()); + stats_.transaction.busy_percentile.Account(std::chrono::duration_cast(duration).count()); } void Pool::AccountQueryError() noexcept { ++stats_.transaction.error_execute_total; } @@ -128,8 +169,10 @@ void Pool::AccountTransactionStarted() noexcept { ++stats_.transaction.total; } void Pool::AccountTransactionCommit(std::chrono::microseconds total_duration, std::chrono::microseconds busy_duration) noexcept { ++stats_.transaction.commit_total; - stats_.transaction.total_percentile.Account(total_duration.count()); - stats_.transaction.busy_percentile.Account(busy_duration.count()); + stats_.transaction.total_percentile + .Account(std::chrono::duration_cast(total_duration).count()); + stats_.transaction.busy_percentile + .Account(std::chrono::duration_cast(busy_duration).count()); } void Pool::AccountTransactionRollback() noexcept { ++stats_.transaction.rollback_total; } diff --git a/odbc/src/storages/odbc/detail/pool.hpp b/odbc/src/storages/odbc/detail/pool.hpp index 5e4b6e964699..554104e2dc8a 100644 --- a/odbc/src/storages/odbc/detail/pool.hpp +++ b/odbc/src/storages/odbc/detail/pool.hpp @@ -9,6 +9,8 @@ #include #include #include +#include +#include #include USERVER_NAMESPACE_BEGIN @@ -17,8 +19,18 @@ namespace storages::odbc::detail { class Pool final : public drivers::impl::ConnectionPoolBase { public: - Pool(const std::string& dsn, std::size_t min_pool_size, std::size_t max_pool_size); - Pool(std::vector dsns, std::size_t min_pool_size, std::size_t max_pool_size); + Pool( + const std::string& dsn, + std::size_t min_pool_size, + std::size_t max_pool_size, + engine::TaskProcessor& blocking_task_processor + ); + Pool( + std::vector dsns, + std::size_t min_pool_size, + std::size_t max_pool_size, + engine::TaskProcessor& blocking_task_processor + ); ~Pool(); @@ -42,6 +54,7 @@ class Pool final : public drivers::impl::ConnectionPoolBase { friend class drivers::impl::ConnectionPoolBase; ConnectionUniquePtr DoCreateConnection(engine::Deadline deadline); + void RunSizeMonitor(); void AccountConnectionCreated() noexcept; void AccountConnectionAcquired() noexcept; @@ -50,8 +63,11 @@ class Pool final : public drivers::impl::ConnectionPoolBase { void AccountOverload() noexcept; const std::vector dsns_; + const std::size_t min_pool_size_; const std::size_t max_pool_size_; + engine::TaskProcessor& blocking_task_processor_; mutable std::atomic dsn_index_{0}; + utils::PeriodicTask size_monitor_; InstanceStatistics stats_{}; }; diff --git a/odbc/src/storages/odbc/detail/result_wrapper.cpp b/odbc/src/storages/odbc/detail/result_wrapper.cpp index e929dbec432d..6a290a15dea0 100644 --- a/odbc/src/storages/odbc/detail/result_wrapper.cpp +++ b/odbc/src/storages/odbc/detail/result_wrapper.cpp @@ -1,182 +1,103 @@ -#include -#include -#include #include + +#include + #include +#include +#include USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail { -namespace { - -void CheckStatus(SQLRETURN ret, SQLHANDLE handle, SQLSMALLINT type) { - if (ret == SQL_SUCCESS) { - return; - } - auto exception_message = fmt::format("SQLFunctionFailed failed: {} {}", ret, GetSQLDiagString(handle, type)); - throw ResultSetError(std::move(exception_message)); -} - -void DestroyResultHandle(SQLHSTMT handle) { - if (handle != SQL_NULL_HSTMT) { - SQLFreeHandle(SQL_HANDLE_STMT, handle); - } -} -} // namespace -ResultWrapper::ResultHandle MakeResultHandle(SQLHDBC handle) { - SQLHSTMT stmt = nullptr; - SQLRETURN ret = SQLAllocHandle(SQL_HANDLE_STMT, handle, &stmt); - if (!SQL_SUCCEEDED(ret)) { - throw ResultSetError("Failed to allocate statement handle"); - } - auto result_handle = ResultWrapper::ResultHandle{stmt, &DestroyResultHandle}; - - ret = - SQLSetStmtAttr(result_handle.get(), SQL_ATTR_CURSOR_TYPE, reinterpret_cast(SQL_CURSOR_DYNAMIC), 0); - if (!SQL_SUCCEEDED(ret)) { - throw ResultSetError("Failed to set cursor type"); - } - - return result_handle; -} - -ResultWrapper::ResultWrapper(ResultHandle&& res) - : handle{std::move(res)} +ResultWrapper::ResultWrapper(std::vector columns, std::vector rows, std::size_t rows_affected) + : columns_{std::move(columns)}, + rows_{std::move(rows)}, + rows_affected_{rows_affected} {} -ResultWrapper::~ResultWrapper() = default; -ResultWrapper::ResultWrapper(ResultWrapper&& other) noexcept = default; +std::size_t ResultWrapper::RowCount() const noexcept { return rows_.size(); } -SQLRETURN ResultWrapper::GetStatus() const { - SQLRETURN ret = SQLMoreResults(handle.get()); - return ret; -} +std::size_t ResultWrapper::FieldCount() const noexcept { return columns_.size(); } -// NOLINTNEXTLINE(readability-make-member-function-const) -void ResultWrapper::Fetch() { - SQLRETURN ret = SQLFetch(handle.get()); - // SQL_NO_DATA means no rows — not an error (e.g. SELECT with empty result set) - if (ret == SQL_NO_DATA) { - return; - } - CheckStatus(ret, handle.get(), SQL_HANDLE_STMT); -} +std::size_t ResultWrapper::RowsAffected() const noexcept { return rows_affected_; } -std::size_t ResultWrapper::RowCount() const { - // TODO: drivers may return -1 or 0 if rows are not fetched yet, overall implementation for select is - // driver-dependent, needs checking - SQLLEN row_count = 0; - CheckStatus(SQLRowCount(handle.get(), &row_count), handle.get(), SQL_HANDLE_STMT); - return static_cast(row_count); +const std::string& ResultWrapper::GetFieldName(std::size_t col) const { + if (col >= columns_.size()) { + throw FieldIndexOutOfBounds{col}; + } + return columns_[col].name; } -std::size_t ResultWrapper::FieldCount() const { - SQLSMALLINT field_count = 0; - CheckStatus(SQLNumResultCols(handle.get(), &field_count), handle.get(), SQL_HANDLE_STMT); - return static_cast(field_count); +SQLSMALLINT ResultWrapper::GetColumnType(std::size_t col) const { + if (col >= columns_.size()) { + throw FieldIndexOutOfBounds{col}; + } + return columns_[col].type; } -std::size_t ResultWrapper::RowsAffected() const { return RowCount(); } - -std::string ResultWrapper::GetFieldName(std::size_t col) const { - SQLCHAR name[1024]; - SQLLEN name_len = sizeof(name); - CheckStatus( - SQLDescribeCol(handle.get(), col + 1, name, sizeof(name), nullptr, nullptr, nullptr, nullptr, nullptr), - handle.get(), - SQL_HANDLE_STMT - ); - return std::string(reinterpret_cast(name), name_len); +const ResultWrapper::Cell& ResultWrapper::GetCell(std::size_t row, std::size_t col) const { + if (row >= rows_.size()) { + throw RowIndexOutOfBounds{row}; + } + if (col >= columns_.size()) { + throw FieldIndexOutOfBounds{col}; + } + return rows_[row][col]; } -SQLSMALLINT ResultWrapper::GetColumnType(std::size_t col) const { - SQLSMALLINT type = 0; - CheckStatus( - SQLDescribeCol(handle.get(), col + 1, nullptr, 0, nullptr, &type, nullptr, nullptr, nullptr), - handle.get(), - SQL_HANDLE_STMT - ); - return type; +const std::string& ResultWrapper::GetValue(std::size_t row, std::size_t col) const { + const auto& cell = GetCell(row, col); + if (!cell.value) { + throw ResultSetError(fmt::format("Field at row {}, column {} is NULL", row, col)); + } + return *cell.value; } -std::string ResultWrapper::GetString(std::size_t row, std::size_t col) const { - SQLSMALLINT type = GetColumnType(col); - if (type != SQL_CHAR && type != SQL_VARCHAR && type != SQL_LONGVARCHAR && type != SQL_WCHAR && - type != SQL_WVARCHAR && type != SQL_WLONGVARCHAR) - { - throw ResultSetError(fmt::format("Type mismatch: column {} is not a string type (SQL type: {})", col + 1, type) - ); +std::string ResultWrapper::GetString(std::size_t row, std::size_t col) const { return GetValue(row, col); } + +template +T ParseNumber(const std::string& value, std::size_t row, std::size_t col) { + try { + return utils::FromString(value); + } catch (const utils::FromStringException& ex) { + throw ResultSetError(fmt::format( + "Cannot convert field at row {}, column {} with value '{}' to the requested type: {}", + row, + col, + value, + ex.what() + )); } - CheckStatus(SQLFetchScroll(handle.get(), SQL_FETCH_ABSOLUTE, row + 1), handle.get(), SQL_HANDLE_STMT); - SQLCHAR value[1024]; - SQLLEN value_len = sizeof(value); - CheckStatus( - SQLGetData(handle.get(), col + 1, SQL_C_CHAR, value, sizeof(value), &value_len), - handle.get(), - SQL_HANDLE_STMT - ); - return std::string(reinterpret_cast(value), value_len); } std::int32_t ResultWrapper::GetInt32(std::size_t row, std::size_t col) const { - SQLSMALLINT type = GetColumnType(col); - if (type != SQL_INTEGER && type != SQL_SMALLINT && type != SQL_TINYINT) { - throw ResultSetError(fmt::format("Type mismatch: column {} is not an int32 type (SQL type: {})", col, type)); - } - CheckStatus(SQLFetchScroll(handle.get(), SQL_FETCH_ABSOLUTE, row + 1), handle.get(), SQL_HANDLE_STMT); - SQLINTEGER value = 0; - CheckStatus(SQLGetData(handle.get(), col + 1, SQL_C_SLONG, &value, 0, nullptr), handle.get(), SQL_HANDLE_STMT); - return static_cast(value); + return ParseNumber(GetValue(row, col), row, col); } std::int64_t ResultWrapper::GetInt64(std::size_t row, std::size_t col) const { - SQLSMALLINT type = GetColumnType(col); - if (type != SQL_BIGINT) { - throw ResultSetError(fmt::format("Type mismatch: column {} is not an int64 type (SQL type: {})", col, type)); - } - CheckStatus(SQLFetchScroll(handle.get(), SQL_FETCH_ABSOLUTE, row + 1), handle.get(), SQL_HANDLE_STMT); - SQLBIGINT value = 0; - CheckStatus(SQLGetData(handle.get(), col + 1, SQL_C_SBIGINT, &value, 0, nullptr), handle.get(), SQL_HANDLE_STMT); - return static_cast(value); + return ParseNumber(GetValue(row, col), row, col); } double ResultWrapper::GetDouble(std::size_t row, std::size_t col) const { - SQLSMALLINT type = GetColumnType(col); - if (type != SQL_DOUBLE && type != SQL_FLOAT && type != SQL_REAL && type != SQL_DECIMAL && type != SQL_NUMERIC) { - throw ResultSetError( - fmt::format("Type mismatch: column {} is not a floating-point type (SQL type: {})", col, type) - ); - } - CheckStatus(SQLFetchScroll(handle.get(), SQL_FETCH_ABSOLUTE, row + 1), handle.get(), SQL_HANDLE_STMT); - SQLDOUBLE value = 0; - CheckStatus(SQLGetData(handle.get(), col + 1, SQL_C_DOUBLE, &value, 0, nullptr), handle.get(), SQL_HANDLE_STMT); - return static_cast(value); + return ParseNumber(GetValue(row, col), row, col); } bool ResultWrapper::GetBool(std::size_t row, std::size_t col) const { - // NOTE: typecheck for bools omitted, since drivers may handle bools differently, but always support SQL_C_BIT - CheckStatus(SQLFetchScroll(handle.get(), SQL_FETCH_ABSOLUTE, row + 1), handle.get(), SQL_HANDLE_STMT); - bool value = false; - CheckStatus( - SQLGetData(handle.get(), col + 1, SQL_C_BIT, &value, sizeof(value), nullptr), - handle.get(), - SQL_HANDLE_STMT + const auto& value = GetValue(row, col); + const auto ieq = utils::StrIcaseEqual{}; + if (value == "1" || ieq(value, "true") || ieq(value, "t") || ieq(value, "yes") || ieq(value, "on")) { + return true; + } + if (value == "0" || ieq(value, "false") || ieq(value, "f") || ieq(value, "no") || ieq(value, "off")) { + return false; + } + throw ResultSetError( + fmt::format("Cannot convert field at row {}, column {} with value '{}' to bool", row, col, value) ); - return value; } -bool ResultWrapper::IsFieldNull(std::size_t row, std::size_t col) const { - CheckStatus(SQLFetchScroll(handle.get(), SQL_FETCH_ABSOLUTE, row + 1), handle.get(), SQL_HANDLE_STMT); - SQLLEN marker = 0; - bool dummy = false; // NOTE: odbc requires a buffer for SQL_C_DEFAULT - CheckStatus( - SQLGetData(handle.get(), col + 1, SQL_C_DEFAULT, &dummy, sizeof(dummy), &marker), - handle.get(), - SQL_HANDLE_STMT - ); - return marker == SQL_NULL_DATA; -} +bool ResultWrapper::IsFieldNull(std::size_t row, std::size_t col) const { return !GetCell(row, col).value; } } // namespace storages::odbc::detail diff --git a/odbc/src/storages/odbc/detail/result_wrapper.hpp b/odbc/src/storages/odbc/detail/result_wrapper.hpp index 3c51bbea8b95..584838ca2581 100644 --- a/odbc/src/storages/odbc/detail/result_wrapper.hpp +++ b/odbc/src/storages/odbc/detail/result_wrapper.hpp @@ -1,51 +1,56 @@ #pragma once -#include -#include -#include - #include -#include +#include #include +#include + +#include USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail { -class ResultWrapper { +/// Fully materialized ODBC result. No ODBC handle is retained: this makes a +/// ResultSet independent from the connection and safe to read after the +/// connection has returned to its pool. +class ResultWrapper final { public: - using ResultHandle = std::unique_ptr, void (*)(SQLHSTMT)>; + struct Column final { + std::string name; + SQLSMALLINT type{}; + }; - ResultWrapper(ResultHandle&& res); - ~ResultWrapper(); + struct Cell final { + std::optional value; + }; - ResultWrapper(const ResultWrapper&) = delete; - ResultWrapper(ResultWrapper&& other) noexcept; + using Row = std::vector; - void Fetch(); + ResultWrapper(std::vector columns, std::vector rows, std::size_t rows_affected); - SQLRETURN GetStatus() const; + std::size_t RowCount() const noexcept; + std::size_t FieldCount() const noexcept; + std::size_t RowsAffected() const noexcept; - std::size_t RowCount() const; - std::size_t FieldCount() const; - std::size_t RowsAffected() const; - - std::string GetFieldName(std::size_t col) const; + const std::string& GetFieldName(std::size_t col) const; SQLSMALLINT GetColumnType(std::size_t col) const; - // Data access methods std::string GetString(std::size_t row, std::size_t col) const; std::int32_t GetInt32(std::size_t row, std::size_t col) const; std::int64_t GetInt64(std::size_t row, std::size_t col) const; double GetDouble(std::size_t row, std::size_t col) const; bool GetBool(std::size_t row, std::size_t col) const; - bool IsFieldNull(std::size_t row, std::size_t col) const; - ResultHandle handle; -}; +private: + const Cell& GetCell(std::size_t row, std::size_t col) const; + const std::string& GetValue(std::size_t row, std::size_t col) const; -ResultWrapper::ResultHandle MakeResultHandle(SQLHDBC); + std::vector columns_; + std::vector rows_; + std::size_t rows_affected_{0}; +}; } // namespace storages::odbc::detail diff --git a/odbc/src/storages/odbc/detail/statistics.hpp b/odbc/src/storages/odbc/detail/statistics.hpp index 7473d845494f..3f77eba8c61b 100644 --- a/odbc/src/storages/odbc/detail/statistics.hpp +++ b/odbc/src/storages/odbc/detail/statistics.hpp @@ -11,7 +11,7 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail { -// A reduced set of buckets for by-query metrics. +// A reduced set of millisecond buckets for timing metrics. // We try to spare the service's metrics quota by default. constexpr inline double kDefaultHistogramBoundsArray[] = {5, 10, 20, 35, 60, 100, 173, 300, 520, 1000, 3200, 10000, 32000, 100000}; diff --git a/odbc/src/storages/odbc/detail/topology/fixed_primary.cpp b/odbc/src/storages/odbc/detail/topology/fixed_primary.cpp index c7d402cc26bb..08f8a3eedac9 100644 --- a/odbc/src/storages/odbc/detail/topology/fixed_primary.cpp +++ b/odbc/src/storages/odbc/detail/topology/fixed_primary.cpp @@ -14,8 +14,12 @@ std::size_t WrappingIncrement(std::atomic& value, std::size_t mod) } // namespace -FixedPrimary::FixedPrimary(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) - : TopologyBase(settings, resolver), +FixedPrimary::FixedPrimary( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor +) + : TopologyBase(settings, resolver, blocking_task_processor), primary_{InitializePrimaryPoolReference()}, secondaries_{InitializeSecondariesVector()} {} diff --git a/odbc/src/storages/odbc/detail/topology/fixed_primary.hpp b/odbc/src/storages/odbc/detail/topology/fixed_primary.hpp index 0af46871a16a..06538af3fefb 100644 --- a/odbc/src/storages/odbc/detail/topology/fixed_primary.hpp +++ b/odbc/src/storages/odbc/detail/topology/fixed_primary.hpp @@ -11,7 +11,11 @@ namespace storages::odbc::detail::topology { class FixedPrimary final : public TopologyBase { public: - FixedPrimary(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver); + FixedPrimary( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor + ); ~FixedPrimary() final; private: diff --git a/odbc/src/storages/odbc/detail/topology/standalone.cpp b/odbc/src/storages/odbc/detail/topology/standalone.cpp index 8a7b8a0c838c..bbedbb933baa 100644 --- a/odbc/src/storages/odbc/detail/topology/standalone.cpp +++ b/odbc/src/storages/odbc/detail/topology/standalone.cpp @@ -8,8 +8,12 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail::topology { -Standalone::Standalone(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) - : TopologyBase(settings, resolver), +Standalone::Standalone( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor +) + : TopologyBase(settings, resolver, blocking_task_processor), pool_{InitializePoolReference()} {} diff --git a/odbc/src/storages/odbc/detail/topology/standalone.hpp b/odbc/src/storages/odbc/detail/topology/standalone.hpp index c730714a9c30..291c6e65a23c 100644 --- a/odbc/src/storages/odbc/detail/topology/standalone.hpp +++ b/odbc/src/storages/odbc/detail/topology/standalone.hpp @@ -8,7 +8,11 @@ namespace storages::odbc::detail::topology { class Standalone final : public TopologyBase { public: - Standalone(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver); + Standalone( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor + ); ~Standalone() final; private: diff --git a/odbc/src/storages/odbc/detail/topology/topology_base.cpp b/odbc/src/storages/odbc/detail/topology/topology_base.cpp index 10dd48944a03..9e14f8989ad9 100644 --- a/odbc/src/storages/odbc/detail/topology/topology_base.cpp +++ b/odbc/src/storages/odbc/detail/topology/topology_base.cpp @@ -26,13 +26,18 @@ std::string ResolveDsn(const std::string& dsn_str, clients::dns::Resolver* resol } // namespace -TopologyBase::TopologyBase(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver) { +TopologyBase::TopologyBase( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor +) { UASSERT(!settings.pools.empty()); pools_.reserve(settings.pools.size()); for (const auto& host : settings.pools) { auto resolved_dsn = ResolveDsn(host.dsn, resolver); - pools_.push_back(std::make_shared(resolved_dsn, host.pool.min_size, host.pool.max_size)); + pools_.push_back(std::make_shared< + Pool>(resolved_dsn, host.pool.min_size, host.pool.max_size, blocking_task_processor)); } } @@ -40,15 +45,16 @@ TopologyBase::~TopologyBase() = default; std::shared_ptr TopologyBase::Create( const settings::ODBCClusterSettings& settings, - clients::dns::Resolver* resolver + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor ) { UASSERT(!settings.pools.empty()); if (settings.pools.size() == 1) { - return std::make_shared(settings, resolver); + return std::make_shared(settings, resolver, blocking_task_processor); } - return std::make_shared(settings, resolver); + return std::make_shared(settings, resolver, blocking_task_processor); } Pool& TopologyBase::SelectPool(ClusterHostType host_type) const { diff --git a/odbc/src/storages/odbc/detail/topology/topology_base.hpp b/odbc/src/storages/odbc/detail/topology/topology_base.hpp index d4c710a5af90..dd8c1179e796 100644 --- a/odbc/src/storages/odbc/detail/topology/topology_base.hpp +++ b/odbc/src/storages/odbc/detail/topology/topology_base.hpp @@ -5,6 +5,7 @@ #include #include +#include #include #include @@ -24,7 +25,8 @@ class TopologyBase { static std::shared_ptr Create( const settings::ODBCClusterSettings& settings, - clients::dns::Resolver* resolver + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor ); Pool& SelectPool(ClusterHostType host_type) const; @@ -32,7 +34,11 @@ class TopologyBase { void WriteStatistics(utils::statistics::Writer& writer) const; protected: - TopologyBase(const settings::ODBCClusterSettings& settings, clients::dns::Resolver* resolver); + TopologyBase( + const settings::ODBCClusterSettings& settings, + clients::dns::Resolver* resolver, + engine::TaskProcessor& blocking_task_processor + ); virtual Pool& GetPrimary() const = 0; virtual Pool& GetSecondary() const = 0; diff --git a/odbc/src/storages/odbc/detail/tracing.cpp b/odbc/src/storages/odbc/detail/tracing.cpp index 29a0f16e653b..f15c7f702117 100644 --- a/odbc/src/storages/odbc/detail/tracing.cpp +++ b/odbc/src/storages/odbc/detail/tracing.cpp @@ -72,6 +72,22 @@ std::string MakeQuerySpanName(std::string_view statement) { return "odbc_query"; } +QuerySpanTags MakeQuerySpanTags(const Query& query) noexcept { + if (const auto name = query.GetOptionalNameView()) { + return { + .statement_name = std::string_view{*name}, + .statement = std::nullopt, + }; + } + if (query.GetLogMode() == Query::LogMode::kFull) { + return { + .statement_name = std::nullopt, + .statement = std::string_view{query.GetStatementView()}, + }; + } + return {}; +} + } // namespace storages::odbc::detail::tracing USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/detail/tracing.hpp b/odbc/src/storages/odbc/detail/tracing.hpp index 39f55bfde22f..c84619b4a209 100644 --- a/odbc/src/storages/odbc/detail/tracing.hpp +++ b/odbc/src/storages/odbc/detail/tracing.hpp @@ -1,14 +1,24 @@ #pragma once +#include #include #include +#include + USERVER_NAMESPACE_BEGIN namespace storages::odbc::detail::tracing { +struct QuerySpanTags final { + std::optional statement_name; + std::optional statement; +}; + std::string MakeQuerySpanName(std::string_view statement); +QuerySpanTags MakeQuerySpanTags(const Query& query) noexcept; + } // namespace storages::odbc::detail::tracing USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/dsn.cpp b/odbc/src/storages/odbc/dsn.cpp index 848ebf561028..d75bc98fd568 100644 --- a/odbc/src/storages/odbc/dsn.cpp +++ b/odbc/src/storages/odbc/dsn.cpp @@ -2,8 +2,9 @@ #include #include -#include +#include #include +#include #include #include @@ -25,51 +26,117 @@ constexpr std::string_view kPasswordKeys[] = {"PWD", "PASSWORD"}; using KeyValueMap = std::unordered_map; -KeyValueMap ParseDsnToMap(std::string_view dsn) { - KeyValueMap result; +struct DsnPart final { + std::string_view raw; + std::string key; + std::string value; + std::size_t value_begin{0}; + std::size_t value_end{0}; + bool valid{false}; +}; + +std::string_view Trim(std::string_view value) { + while (!value.empty() && std::isspace(static_cast(value.front()))) { + value.remove_prefix(1); + } + while (!value.empty() && std::isspace(static_cast(value.back()))) { + value.remove_suffix(1); + } + return value; +} - std::string current_key; - std::string current_value; +std::vector SplitDsn(std::string_view dsn) { + std::vector result; + std::size_t part_begin = 0; bool in_braces = false; - bool reading_value = false; + bool saw_equal = false; + bool saw_value = false; - for (char c : dsn) { - if (c == '{' && !in_braces && reading_value) { - in_braces = true; + for (std::size_t index = 0; index < dsn.size(); ++index) { + const auto c = dsn[index]; + if (!saw_equal && c == '=') { + saw_equal = true; continue; } - - if (c == '}' && in_braces) { - in_braces = false; + if (saw_equal && !saw_value && !std::isspace(static_cast(c))) { + saw_value = true; + in_braces = c == '{'; continue; } - - if (c == '=' && !in_braces && !reading_value) { - reading_value = true; + if (in_braces && c == '}') { + if (index + 1 < dsn.size() && dsn[index + 1] == '}') { + ++index; + } else { + in_braces = false; + } continue; } + if (!in_braces && c == ';') { + result.push_back(dsn.substr(part_begin, index - part_begin + 1)); + part_begin = index + 1; + saw_equal = false; + saw_value = false; + } + } + if (part_begin < dsn.size()) { + result.push_back(dsn.substr(part_begin)); + } + return result; +} - if (c == ';' && !in_braces) { - if (!current_key.empty()) { - result[utils::text::ToUpper(current_key)] = current_value; +std::string DecodeValue(std::string_view raw_value) { + const auto trimmed = Trim(raw_value); + if (trimmed.size() >= 2 && trimmed.front() == '{' && trimmed.back() == '}') { + std::string decoded; + decoded.reserve(trimmed.size() - 2); + for (std::size_t index = 1; index + 1 < trimmed.size(); ++index) { + if (trimmed[index] == '}' && index + 1 < trimmed.size() - 1 && trimmed[index + 1] == '}') { + ++index; } - current_key.clear(); - current_value.clear(); - reading_value = false; - continue; + decoded += trimmed[index]; } + return decoded; + } + return std::string{raw_value}; +} - if (reading_value) { - current_value += c; - } else { - current_key += c; - } +DsnPart ParsePart(std::string_view raw) { + const auto content_end = !raw.empty() && raw.back() == ';' ? raw.size() - 1 : raw.size(); + const auto equal = raw.substr(0, content_end).find('='); + if (equal == std::string_view::npos) { + return {.raw = raw}; } - if (!current_key.empty()) { - result[utils::text::ToUpper(current_key)] = current_value; + const auto key = Trim(raw.substr(0, equal)); + if (key.empty()) { + return {.raw = raw}; } + const auto value_begin = equal + 1; + return { + .raw = raw, + .key = utils::text::ToUpper(std::string{key}), + .value = DecodeValue(raw.substr(value_begin, content_end - value_begin)), + .value_begin = value_begin, + .value_end = content_end, + .valid = true, + }; +} +std::vector ParseDsnParts(std::string_view dsn) { + std::vector result; + for (const auto raw : SplitDsn(dsn)) { + result.push_back(ParsePart(raw)); + } + return result; +} + +KeyValueMap ParseDsnToMap(std::string_view dsn) { + KeyValueMap result; + for (auto& part : ParseDsnParts(dsn)) { + if (part.valid) { + result[std::move(part.key)] = std::move(part.value); + } + } return result; } @@ -83,30 +150,14 @@ std::string FindValue(const KeyValueMap& map, const auto& keys) { return {}; } -std::string RebuildDsn(const KeyValueMap& map) { - std::string result; - result.reserve(128); - for (const auto& [key, value] : map) { - if (!result.empty()) { - result += ';'; - } - - const bool needs_braces = - (value.find(';') != std::string::npos || value.find('{') != std::string::npos || - value.find('}') != std::string::npos); - - result += key; - result += '='; - if (needs_braces) { - result += '{'; - } - result += value; - if (needs_braces) { - result += '}'; - } - } +bool IsOneOf(std::string_view key, const auto& keys) { + return std::find(std::begin(keys), std::end(keys), key) != std::end(keys); +} - return result; +void AppendWithValue(std::string& output, const DsnPart& part, std::string_view value) { + output.append(part.raw.substr(0, part.value_begin)); + output.append(value); + output.append(part.raw.substr(part.value_end)); } } // namespace @@ -136,26 +187,27 @@ std::string GetHostPort(const Dsn& dsn) { } std::string DsnCutPassword(const Dsn& dsn) { - auto map = ParseDsnToMap(dsn.GetUnderlying()); - - for (const auto& key : kPasswordKeys) { - map.erase(std::string{key}); + std::string result; + result.reserve(dsn.GetUnderlying().size()); + for (const auto& part : ParseDsnParts(dsn.GetUnderlying())) { + if (!part.valid || !IsOneOf(part.key, kPasswordKeys)) { + result.append(part.raw); + } } - - return RebuildDsn(map); + return result; } std::string DsnMaskPassword(const Dsn& dsn) { - auto map = ParseDsnToMap(dsn.GetUnderlying()); - - for (const auto& key : kPasswordKeys) { - auto it = map.find(std::string{key}); - if (it != map.end()) { - it->second = "***"; + std::string result; + result.reserve(dsn.GetUnderlying().size()); + for (const auto& part : ParseDsnParts(dsn.GetUnderlying())) { + if (part.valid && IsOneOf(part.key, kPasswordKeys)) { + AppendWithValue(result, part, "***"); + } else { + result.append(part.raw); } } - - return RebuildDsn(map); + return result; } bool IsIpAddress(std::string_view host) { @@ -187,18 +239,8 @@ bool IsIpAddress(std::string_view host) { } Dsn ResolveDsnHost(const Dsn& dsn, clients::dns::Resolver& resolver, engine::Deadline deadline) { - auto map = ParseDsnToMap(dsn.GetUnderlying()); - - std::string server; - std::string server_key; - for (const auto& key : kServerKeys) { - auto it = map.find(std::string{key}); - if (it != map.end() && !it->second.empty()) { - server = it->second; - server_key = std::string{key}; - break; - } - } + const auto opts = ParseDsn(dsn); + const auto& server = opts.server; if (server.empty() || IsIpAddress(server)) { return dsn; @@ -212,11 +254,42 @@ Dsn ResolveDsnHost(const Dsn& dsn, clients::dns::Resolver& resolver, engine::Dea auto resolved_ip = addrs.front().PrimaryAddressString(); LOG_DEBUG() << "Resolved ODBC host " << server << " to " << resolved_ip; - map[server_key] = resolved_ip; + return detail::ReplaceDsnHost(dsn, resolved_ip); +} + +namespace detail { + +Dsn ReplaceDsnHost(const Dsn& dsn, std::string_view resolved_host) { + const auto parts = ParseDsnParts(dsn.GetUnderlying()); + std::optional target; + for (const auto& key : kServerKeys) { + for (std::size_t index = 0; index < parts.size(); ++index) { + if (parts[index].valid && parts[index].key == key && !parts[index].value.empty()) { + target = index; + } + } + if (target) { + break; + } + } + if (!target) { + return dsn; + } - return Dsn{RebuildDsn(map)}; + std::string result; + result.reserve(dsn.GetUnderlying().size() + resolved_host.size()); + for (std::size_t index = 0; index < parts.size(); ++index) { + if (index == *target) { + AppendWithValue(result, parts[index], resolved_host); + } else { + result.append(parts[index].raw); + } + } + return Dsn{std::move(result)}; } +} // namespace detail + } // namespace storages::odbc USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/dsn.hpp b/odbc/src/storages/odbc/dsn.hpp index ed04ed640b5a..8e7463cab007 100644 --- a/odbc/src/storages/odbc/dsn.hpp +++ b/odbc/src/storages/odbc/dsn.hpp @@ -33,6 +33,13 @@ bool IsIpAddress(std::string_view host); Dsn ResolveDsnHost(const Dsn& dsn, clients::dns::Resolver& resolver, engine::Deadline deadline); +namespace detail { + +/// Replaces only the effective host value while preserving all other DSN text. +Dsn ReplaceDsnHost(const Dsn& dsn, std::string_view resolved_host); + +} // namespace detail + } // namespace storages::odbc USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/exception.cpp b/odbc/src/storages/odbc/exception.cpp index 33f16bc91bcc..e2ef551103e6 100644 --- a/odbc/src/storages/odbc/exception.cpp +++ b/odbc/src/storages/odbc/exception.cpp @@ -6,6 +6,28 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc { +RuntimeError::RuntimeError(std::string message, std::vector diagnostics, bool invalid_handle) + : Error(std::move(message)), + diagnostics_(std::move(diagnostics)), + invalid_handle_(invalid_handle) +{} + +const std::vector& RuntimeError::GetDiagnostics() const noexcept { return diagnostics_; } + +bool RuntimeError::HasSqlStateClass(std::string_view sql_state_class) const noexcept { + if (sql_state_class.size() != 2) { + return false; + } + for (const auto& diagnostic : diagnostics_) { + if (diagnostic.sql_state.size() >= 2 && diagnostic.sql_state.compare(0, 2, sql_state_class) == 0) { + return true; + } + } + return false; +} + +bool RuntimeError::IsInvalidHandle() const noexcept { return invalid_handle_; } + FieldIndexOutOfBounds::FieldIndexOutOfBounds(std::size_t index) : ResultSetError(fmt::format("Field index {} is out of bounds", index)) {} diff --git a/odbc/src/storages/odbc/odbc_secdist.cpp b/odbc/src/storages/odbc/odbc_secdist.cpp index df718b276f34..36f01e6bdba9 100644 --- a/odbc/src/storages/odbc/odbc_secdist.cpp +++ b/odbc/src/storages/odbc/odbc_secdist.cpp @@ -34,9 +34,16 @@ OdbcSettings::OdbcSettings(const formats::json::Value& doc) { storages::secdist::CheckIsObject(db_config, dbalias); std::vector connections; + const auto has_dsn = db_config.HasMember("dsn"); + const auto has_hosts = db_config.HasMember("hosts"); + if (has_dsn == has_hosts) { + throw storages::secdist::SecdistError( + fmt::format("Database '{}' must have exactly one of 'dsn' or 'hosts'", dbalias) + ); + } // Support both single DSN and array of DSNs - if (db_config.HasMember("dsn")) { + if (has_dsn) { // Single DSN format OdbcConnectionInfo info; info.dsn = db_config["dsn"].As(); @@ -54,6 +61,11 @@ OdbcSettings::OdbcSettings(const formats::json::Value& doc) { connections.push_back(std::move(info)); } else if (host_it->IsObject()) { // Object with dsn field + if (host_it->GetSize() != 1 || !host_it->HasMember("dsn")) { + throw storages::secdist::SecdistError( + fmt::format("Database '{}': each host object must contain exactly one 'dsn' field", dbalias) + ); + } OdbcConnectionInfo info; info.dsn = (*host_it)["dsn"].As(); connections.push_back(std::move(info)); @@ -61,10 +73,6 @@ OdbcSettings::OdbcSettings(const formats::json::Value& doc) { storages::secdist::ThrowInvalidSecdistType(*host_it, "a string or object"); } } - } else { - throw storages::secdist::SecdistError( - fmt::format("Database '{}' must have either 'dsn' or 'hosts' field", dbalias) - ); } if (connections.empty()) { diff --git a/odbc/src/storages/odbc/result_set.cpp b/odbc/src/storages/odbc/result_set.cpp index 0eee3d40efd4..3593ac003f3a 100644 --- a/odbc/src/storages/odbc/result_set.cpp +++ b/odbc/src/storages/odbc/result_set.cpp @@ -13,6 +13,15 @@ ResultSet::size_type ResultSet::Size() const { return pimpl_ != nullptr ? pimpl_ ResultSet::size_type ResultSet::FieldCount() const { return pimpl_ != nullptr ? pimpl_->FieldCount() : 0; } +ResultSet::size_type ResultSet::RowsAffected() const { return pimpl_ != nullptr ? pimpl_->RowsAffected() : 0; } + +std::string_view ResultSet::GetFieldName(size_type index) const { + if (!pimpl_) { + throw FieldIndexOutOfBounds{index}; + } + return pimpl_->GetFieldName(index); +} + bool ResultSet::IsEmpty() const { return Size() == 0; } ResultSet::reference ResultSet::operator[](size_type index) const& { diff --git a/odbc/src/storages/odbc/transaction.cpp b/odbc/src/storages/odbc/transaction.cpp index 67b5e6defbb1..7a67f162972a 100644 --- a/odbc/src/storages/odbc/transaction.cpp +++ b/odbc/src/storages/odbc/transaction.cpp @@ -8,6 +8,7 @@ #include #include +#include #include #include #include @@ -17,16 +18,24 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc { -Transaction::Transaction(detail::ConnectionPtr&& connection, detail::Pool& pool, engine::Deadline deadline) +Transaction::Transaction( + detail::ConnectionPtr&& connection, + detail::Pool& pool, + std::chrono::milliseconds network_timeout, + std::chrono::milliseconds statement_timeout +) : connection_{std::move(connection)}, pool_{&pool}, - deadline_{deadline}, + network_timeout_{network_timeout}, + statement_timeout_{statement_timeout}, start_time_{utils::datetime::SteadyCoarseClock::now()}, busy_time_{0}, span_{storages::odbc::impl::tracing::kTransactionSpan} { - detail::CheckDeadlineNotExpired(deadline_); - (*connection_)->Begin(deadline_); + const auto deadline = + std::min(detail::GetExecuteDeadline(network_timeout_), detail::GetExecuteDeadline(statement_timeout_)); + detail::CheckDeadlineNotExpired(deadline); + (*connection_)->Begin(deadline); trx_lock_.Lock(); pool_->AccountTransactionStarted(); } @@ -35,11 +44,16 @@ Transaction::Transaction(Transaction&& other) noexcept = default; Transaction::~Transaction() { if (connection_->IsValid()) { + const engine::TaskCancellationBlocker cancellation_blocker; try { - (*connection_)->Rollback(deadline_); + (*connection_)->Rollback(engine::Deadline::FromDuration(detail::kDefaultCleanupTimeout)); pool_->AccountTransactionRollback(); } catch (const std::exception& ex) { + (*connection_)->NotifyBroken(); LOG_ERROR() << "Failed to auto rollback a transaction: " << ex.what(); + } catch (...) { + (*connection_)->NotifyBroken(); + LOG_ERROR() << "Failed to auto rollback a transaction with an unknown exception"; } trx_lock_.Unlock(); } @@ -48,9 +62,11 @@ Transaction::~Transaction() { void Transaction::Commit() { const utils::FastScopeGuard unlock_guard([this]() noexcept { trx_lock_.Unlock(); }); AssertValid(); - detail::CheckDeadlineNotExpired(deadline_); + const auto deadline = + std::min(detail::GetExecuteDeadline(network_timeout_), detail::GetExecuteDeadline(statement_timeout_)); + detail::CheckDeadlineNotExpired(deadline); auto connection = std::move(*connection_); - connection->Commit(deadline_); + connection->Commit(deadline); const auto total_duration = std::chrono::duration_cast< std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start_time_); @@ -60,9 +76,9 @@ void Transaction::Commit() { void Transaction::Rollback() { const utils::FastScopeGuard unlock_guard([this]() noexcept { trx_lock_.Unlock(); }); AssertValid(); - detail::CheckDeadlineNotExpired(deadline_); + const auto deadline = engine::Deadline::FromDuration(detail::kDefaultCleanupTimeout); auto connection = std::move(*connection_); - connection->Rollback(deadline_); + connection->Rollback(deadline); pool_->AccountTransactionRollback(); } @@ -72,25 +88,25 @@ ResultSet Transaction::DoExecute( const impl::ParameterList& parameters ) { AssertValid(); - detail::CheckDeadlineNotExpired(deadline_); tracing::Span span{storages::odbc::impl::tracing::kExecuteSpan}; - auto statement_deadline = deadline_; + auto network_timeout = network_timeout_; + auto statement_timeout = statement_timeout_; if (command_control) { if (command_control->network_timeout) { - statement_deadline = - std::min(statement_deadline, detail::GetExecuteDeadline(*command_control->network_timeout)); + network_timeout = *command_control->network_timeout; } if (command_control->statement_timeout) { - statement_deadline = - std::min(statement_deadline, detail::GetExecuteDeadline(*command_control->statement_timeout)); + statement_timeout = *command_control->statement_timeout; } } + const auto statement_deadline = + std::min(detail::GetExecuteDeadline(network_timeout), detail::GetExecuteDeadline(statement_timeout)); detail::CheckDeadlineNotExpired(statement_deadline); const auto start = utils::datetime::SteadyCoarseClock::now(); try { - auto result = (*connection_)->Query(query.GetStatementView(), parameters, statement_deadline); + auto result = (*connection_)->Query(query, parameters, statement_deadline); const auto elapsed = std::chrono::duration_cast< std::chrono::microseconds>(utils::datetime::SteadyCoarseClock::now() - start); busy_time_ += elapsed; diff --git a/odbc/tests/odbc_dsn_test.cpp b/odbc/tests/odbc_dsn_test.cpp index b9a28d8fd8da..25dfe63ba80b 100644 --- a/odbc/tests/odbc_dsn_test.cpp +++ b/odbc/tests/odbc_dsn_test.cpp @@ -118,6 +118,37 @@ UTEST(DsnMaskPassword, NoPassword) { EXPECT_THAT(result, testing::Not(testing::HasSubstr("***"))); } +UTEST(DsnTransform, PreservesUnrelatedAttributesExactly) { + const Dsn dsn{ + "DrIvEr={Driver;With}}Brace};UID=user;PWD={se;cr}}et};" + "ApplicationIntent=ReadOnly;SERVER=db.local;Extra={x}};y};" + }; + + EXPECT_EQ( + detail::ReplaceDsnHost(dsn, "127.0.0.42").GetUnderlying(), + "DrIvEr={Driver;With}}Brace};UID=user;PWD={se;cr}}et};" + "ApplicationIntent=ReadOnly;SERVER=127.0.0.42;Extra={x}};y};" + ); + EXPECT_EQ( + DsnMaskPassword(dsn), + "DrIvEr={Driver;With}}Brace};UID=user;PWD=***;" + "ApplicationIntent=ReadOnly;SERVER=db.local;Extra={x}};y};" + ); + EXPECT_EQ( + DsnCutPassword(dsn), + "DrIvEr={Driver;With}}Brace};UID=user;" + "ApplicationIntent=ReadOnly;SERVER=db.local;Extra={x}};y};" + ); +} + +UTEST(DsnTransform, ReplacesOnlyEffectiveServerAlias) { + const Dsn dsn{"SERVER=old-first;Unknown=a;server=old-effective;HOST=fallback"}; + EXPECT_EQ( + detail::ReplaceDsnHost(dsn, "10.0.0.1").GetUnderlying(), + "SERVER=old-first;Unknown=a;server=10.0.0.1;HOST=fallback" + ); +} + UTEST(IsIpAddress, IPv4) { EXPECT_TRUE(IsIpAddress("192.168.1.1")); EXPECT_TRUE(IsIpAddress("10.0.0.1")); diff --git a/odbc/tests/odbc_exceptions_test.cpp b/odbc/tests/odbc_exceptions_test.cpp index 53783ee7af9e..b700dc015b49 100644 --- a/odbc/tests/odbc_exceptions_test.cpp +++ b/odbc/tests/odbc_exceptions_test.cpp @@ -1,6 +1,14 @@ #include + +#include +#include + +#include +#include +#include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -49,9 +57,81 @@ UTEST(StatementError, QueryingUnexistentTable) { } UTEST(StatementError, InvalidSyntax) { - auto cluster = MakeCluster(); + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "CREATE TEMP TABLE odbc_statement_error_session_marker(value INTEGER)" + ); + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "INSERT INTO odbc_statement_error_session_marker(value) VALUES (1)" + ); + + try { + cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELEC 1"); + FAIL() << "Invalid SQL must throw"; + } catch (const storages::odbc::StatementError& ex) { + ASSERT_FALSE(ex.GetDiagnostics().empty()); + EXPECT_EQ(ex.GetDiagnostics().front().sql_state, "42601"); + EXPECT_TRUE(ex.HasSqlStateClass("42")); + EXPECT_FALSE(ex.HasSqlStateClass("08")); + EXPECT_NE(std::string_view{ex.what()}.find("[42601]"), std::string_view::npos); + } + + // A statement-level error must not evict or poison an otherwise healthy HDBC. + const auto result = + cluster + .Execute(storages::odbc::ClusterHostType::kMaster, "SELECT value FROM odbc_statement_error_session_marker"); + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetInt32(), 1); +} + +UTEST(StatementError, ClassifiesConnectionDiagnostics) { + const std::vector diagnostics{ + DiagnosticRecord{.sql_state = "01004", .native_error = 0, .message = "truncated"}, + DiagnosticRecord{.sql_state = "08006", .native_error = 7, .message = "connection failure"}, + }; + + EXPECT_TRUE(detail::HasConnectionError(diagnostics)); + const StatementError error{"driver call failed", diagnostics, false}; + EXPECT_TRUE(error.HasSqlStateClass("08")); + EXPECT_FALSE(error.IsInvalidHandle()); + + const auto formatted = detail::FormatSQLDiagnostics(diagnostics); + EXPECT_NE(formatted.find("[01004] truncated (native code 0)"), std::string::npos); + EXPECT_NE(formatted.find("[08006] connection failure (native code 7)"), std::string::npos); +} + +UTEST(OdbcSecdist, RejectsAmbiguousConnectionSource) { + const auto doc = formats::json::FromString(R"({ + "odbc_settings": {"databases": {"test": { + "dsn": "dsn-1", + "hosts": ["dsn-2"] + }}} + })"); + UEXPECT_THROW(secdist::OdbcSettings{doc}, storages::secdist::SecdistError); +} + +UTEST(OdbcSecdist, RejectsMalformedHostObject) { + const auto doc = formats::json::FromString(R"({ + "odbc_settings": {"databases": {"test": { + "hosts": [{"dsn": "dsn-1", "unexpected": true}] + }}} + })"); + UEXPECT_THROW(secdist::OdbcSettings{doc}, storages::secdist::SecdistError); +} + +UTEST(OdbcSecdist, RejectsEmptyDsn) { + const auto direct = formats::json::FromString(R"({ + "odbc_settings": {"databases": {"test": {"dsn": ""}}} + })"); + UEXPECT_THROW(secdist::OdbcSettings{direct}, storages::secdist::SecdistError); - UEXPECT_THROW(cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELEC 1"), storages::odbc::StatementError); + const auto in_hosts = formats::json::FromString(R"({ + "odbc_settings": {"databases": {"test": {"hosts": [""]}}} + })"); + UEXPECT_THROW(secdist::OdbcSettings{in_hosts}, storages::secdist::SecdistError); } UTEST(StatementError, InvalidColumnReference) { diff --git a/odbc/tests/odbc_postgresql_test.cpp b/odbc/tests/odbc_postgresql_test.cpp index a3228b2c7480..7e2702beb231 100644 --- a/odbc/tests/odbc_postgresql_test.cpp +++ b/odbc/tests/odbc_postgresql_test.cpp @@ -1,6 +1,8 @@ #include #include +#include #include +#include #include #include #include @@ -65,6 +67,54 @@ UTEST(Query, BindsParametersWithoutInterpolation) { EXPECT_FALSE(result[0][6].GetBool()); } +UTEST(Query, RejectsUnsignedValuesOutsidePortableBigintRange) { + auto cluster = MakeCluster(); + + const auto supported = cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "SELECT ?::bigint", + static_cast(std::numeric_limits::max()) + ); + ASSERT_EQ(supported.Size(), 1); + EXPECT_EQ(supported[0][0].GetInt64(), std::numeric_limits::max()); + + UEXPECT_THROW( + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "SELECT ?::numeric", + std::numeric_limits::max() + ), + storages::odbc::StatementError + ); +} + +UTEST(Query, HonorsQueryLogModeAndName) { + const Query named{ + "SELECT 'must not leak'", + Query::Name{"safe_query_name"}, + Query::LogMode::kNameOnly, + }; + const auto named_tags = detail::tracing::MakeQuerySpanTags(named); + ASSERT_TRUE(named_tags.statement_name); + EXPECT_EQ(*named_tags.statement_name, "safe_query_name"); + EXPECT_FALSE(named_tags.statement); + + const Query unnamed_name_only{ + "SELECT 'must not leak either'", + std::nullopt, + Query::LogMode::kNameOnly, + }; + const auto hidden_tags = detail::tracing::MakeQuerySpanTags(unnamed_name_only); + EXPECT_FALSE(hidden_tags.statement_name); + EXPECT_FALSE(hidden_tags.statement); + + const Query unnamed_full{"SELECT 1"}; + const auto full_tags = detail::tracing::MakeQuerySpanTags(unnamed_full); + EXPECT_FALSE(full_tags.statement_name); + ASSERT_TRUE(full_tags.statement); + EXPECT_EQ(*full_tags.statement, "SELECT 1"); +} + UTEST(Query, BindsTypedNull) { auto cluster = MakeCluster(); @@ -155,6 +205,108 @@ UTEST(Query, GetStringFromNumber) { EXPECT_EQ(result[0][0].GetString(), "42"); } +UTEST(Query, MaterializedResultOutlivesConnectionAndSubsequentQuery) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + std::optional saved_result; + { + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + saved_result + .emplace(cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELECT ?::text AS saved_value", "first") + ); + + const auto second = cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELECT ?::integer", 2); + ASSERT_EQ(second.Size(), 1); + EXPECT_EQ(second[0][0].GetInt32(), 2); + } + + ASSERT_TRUE(saved_result); + ASSERT_EQ(saved_result->Size(), 1); + EXPECT_EQ(saved_result->GetFieldName(0), "saved_value"); + EXPECT_EQ((*saved_result)[0][0].GetString(), "first"); +} + +UTEST(Query, MaterializesLongEmptyAndNullValues) { + auto cluster = MakeCluster(); + const std::string long_value(70'000, 'x'); + + const auto result = cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "SELECT ?::text AS long_value, ?::text AS empty_value, NULL::text AS null_value", + long_value, + std::string{} + ); + + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetString(), long_value); + EXPECT_EQ(result[0][1].GetString(), ""); + EXPECT_TRUE(result[0][2].IsNull()); + UEXPECT_THROW(result[0][2].GetString(), storages::odbc::ResultSetError); +} + +UTEST(Query, MaterializesChunkBoundariesAndTypedNulls) { + auto cluster = MakeCluster(); + const std::string value_4095(4095, 'a'); + const std::string value_4096(4096, 'b'); + const std::string value_4097(4097, 'c'); + const std::string value_65537(65'537, 'd'); + + const auto result = cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "SELECT ?::text, ?::text, ?::text, ?::text, NULL::integer, NULL::boolean, NULL::double precision", + value_4095, + value_4096, + value_4097, + value_65537 + ); + + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetString(), value_4095); + EXPECT_EQ(result[0][1].GetString(), value_4096); + EXPECT_EQ(result[0][2].GetString(), value_4097); + EXPECT_EQ(result[0][3].GetString(), value_65537); + for (std::size_t index = 4; index < 7; ++index) { + EXPECT_TRUE(result[0][index].IsNull()); + } + UEXPECT_THROW(result[0][4].GetInt32(), storages::odbc::ResultSetError); + UEXPECT_THROW(result[0][5].GetBool(), storages::odbc::ResultSetError); + UEXPECT_THROW(result[0][6].GetDouble(), storages::odbc::ResultSetError); +} + +UTEST(Query, MaterializedResultSurvivesTopologyReload) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + const auto result = cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELECT ?::text", "before reload"); + + const std::string reloaded_dsn = std::string{kDSN} + "ApplicationName=odbc-materialized-result;"; + cluster.UpdateSettings(storages::odbc::settings::ODBCClusterSettings{{ + storages::odbc::settings::HostSettings{reloaded_dsn, {1, 1}}, + }}); + + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetString(), "before reload"); +} + +UTEST(Query, SeparatesRowsFromRowsAffected) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + cluster.Execute(storages::odbc::ClusterHostType::kMaster, "CREATE TEMP TABLE odbc_rows_affected(value INTEGER)"); + + const auto insert = cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "INSERT INTO odbc_rows_affected(value) VALUES (?), (?)", + 1, + 2 + ); + EXPECT_EQ(insert.Size(), 0); + EXPECT_EQ(insert.RowsAffected(), 2); + + const auto select = + cluster + .Execute(storages::odbc::ClusterHostType::kMaster, "SELECT value FROM odbc_rows_affected ORDER BY value"); + EXPECT_EQ(select.Size(), 2); + EXPECT_EQ(select.RowsAffected(), 0); +} + UTEST(Pool, LessQueriesThanConnections) { std::size_t poolConnections = 5; auto hostSettings = storages::odbc::settings::HostSettings{kDSN, {poolConnections, poolConnections}}; diff --git a/odbc/tests/odbc_round_robin_pool_test.cpp b/odbc/tests/odbc_round_robin_pool_test.cpp index 0c6d44941bab..896e9386134a 100644 --- a/odbc/tests/odbc_round_robin_pool_test.cpp +++ b/odbc/tests/odbc_round_robin_pool_test.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -13,7 +14,7 @@ UTEST(Topology, RoundRobinPoolCreatesConnectionsInTurn) { // We don't need a real database here: all DSNs are invalid, so each Acquire should throw. // This is a smoke-test for the multi-DSN Pool constructor. std::vector dsns{"invalid_dsn_1", "invalid_dsn_2", "invalid_dsn_3"}; - storages::odbc::detail::Pool pool(std::move(dsns), 0, 1); + storages::odbc::detail::Pool pool(std::move(dsns), 0, 1, engine::current_task::GetBlockingTaskProcessor()); for (int i = 0; i < 6; ++i) { UEXPECT_THROW(pool.Acquire(engine::Deadline::FromDuration(std::chrono::seconds{2})), storages::odbc::Error); diff --git a/odbc/tests/odbc_transaction_test.cpp b/odbc/tests/odbc_transaction_test.cpp index 28742d1eca4e..dd2b08070e53 100644 --- a/odbc/tests/odbc_transaction_test.cpp +++ b/odbc/tests/odbc_transaction_test.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -9,6 +10,8 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc::tests { +using namespace std::chrono_literals; + UTEST(CreateTransaction, Works) { auto cluster = MakeCluster(); auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); @@ -54,6 +57,17 @@ UTEST(Transaction, BindsParameters) { trx.Commit(); } +UTEST(Transaction, MaterializedResultRemainsReadableAfterCommit) { + auto cluster = MakeCluster(); + auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); + const auto result = trx.Execute("SELECT ?::text AS value", "materialized"); + trx.Commit(); + + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result.GetFieldName(0), "value"); + EXPECT_EQ(result[0][0].GetString(), "materialized"); +} + UTEST(Transaction, DoubleCommitThrows) { auto cluster = MakeCluster(); auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); @@ -181,6 +195,97 @@ UTEST(Transaction, PlainQueryAfterTransaction) { EXPECT_EQ(result[0][0].GetInt64(), 2); } +UTEST(Transaction, CommandControlIsFreshForEachOperation) { + auto cluster = MakeCluster(); + auto trx = cluster.Begin( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{ + .network_timeout = 50ms, + .statement_timeout = 50ms, + } + ); + + engine::SleepFor(100ms); + const auto result = trx.Execute("SELECT 1"); + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetInt32(), 1); + + engine::SleepFor(100ms); + UEXPECT_NO_THROW(trx.Commit()); +} + +UTEST(Transaction, AutoRollbackHasIndependentCleanupDeadline) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + "CREATE TABLE IF NOT EXISTS t_expired_transaction_cleanup(id INT PRIMARY KEY)" + ); + cluster.Execute(storages::odbc::ClusterHostType::kMaster, "DELETE FROM t_expired_transaction_cleanup"); + + { + auto trx = cluster.Begin( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{ + .network_timeout = 50ms, + .statement_timeout = 50ms, + } + ); + trx.Execute("INSERT INTO t_expired_transaction_cleanup(id) VALUES(1)"); + engine::SleepFor(100ms); + // The transaction command-control lifetime has elapsed, but destructor + // rollback still gets a separate cleanup budget. + } + + const auto result = + cluster.Execute(storages::odbc::ClusterHostType::kMaster, "SELECT COUNT(*) FROM t_expired_transaction_cleanup"); + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetInt64(), 0); +} + +UTEST(PoolDeadline, AcquireTimeoutIsReportedAsOperationInterrupted) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + auto holder = cluster.Begin(storages::odbc::ClusterHostType::kMaster); + + UEXPECT_THROW( + cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{ + .network_timeout = 50ms, + .statement_timeout = 1s, + }, + "SELECT 1" + ), + storages::odbc::OperationInterrupted + ); + holder.Rollback(); +} + +UTEST(PoolDeadline, PoolWaitDoesNotConsumeStatementTimeout) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + auto holder = cluster.Begin(storages::odbc::ClusterHostType::kMaster); + + auto query = utils::Async("odbc-fresh-statement-deadline", [&cluster] { + return cluster.Execute( + storages::odbc::ClusterHostType::kMaster, + storages::odbc::CommandControl{ + .network_timeout = 1s, + .statement_timeout = 50ms, + }, + "SELECT 1" + ); + }); + engine::SleepFor(100ms); + holder.Rollback(); + + const auto result = query.Get(); + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetInt32(), 1); +} + } // namespace storages::odbc::tests USERVER_NAMESPACE_END diff --git a/scripts/docs/en/userver/odbc.md b/scripts/docs/en/userver/odbc.md index c9dedb92f427..e6bb985caf09 100644 --- a/scripts/docs/en/userver/odbc.md +++ b/scripts/docs/en/userver/odbc.md @@ -16,15 +16,22 @@ literals internally. @snippet odbc/tests/odbc_postgresql_test.cpp ODBC parameter binding -The variadic API supports booleans, signed and unsigned integers, floating -point values, strings and string views. Use `std::optional` for a typed -nullable value, or `nullptr` when the ODBC driver can infer the parameter type -from the statement. The number of C++ arguments must match the number of `?` +The variadic API supports booleans, signed integers, unsigned integers up to +`INT64_MAX`, floating point values, strings and string views. Larger unsigned +values are rejected instead of relying on driver-specific conversion outside +the portable SQL `BIGINT` range. Use `std::optional` for a typed nullable +value, or `nullptr` when the ODBC driver can infer the parameter type from the +statement. The number of C++ arguments must match the number of `?` placeholders. `storages::odbc::Cluster::Execute` returns a storages::odbc::ResultSet. Its rows contain storages::odbc::Field values that provide typed getters such as `GetInt32`, `GetInt64`, `GetDouble`, `GetBool`, and `GetString`. +The result is fully materialized as an in-memory snapshot before the connection +returns to the pool, so it remains readable after another query, transaction +completion, or topology reload. Consequently, large or unbounded `SELECT`s use +memory proportional to the complete result. `ResultSet::Size()` is the number +of materialized rows; use `ResultSet::RowsAffected()` for DML row counts. ### Transactions @@ -39,9 +46,11 @@ Parameters are bound in transaction queries in exactly the same way: storages::odbc::CommandControl configures the connection-acquisition/network timeout and statement timeout for an operation. Pass an storages::odbc::OptionalCommandControl to `Cluster::Execute`, `Cluster::Begin`, -or `Transaction::Execute` to override the defaults. The effective deadline is -the earliest of the network operation budget, statement timeout, transaction -deadline, and task-inherited request deadline. +or `Transaction::Execute` to override the defaults. `Begin`, every transaction +`Execute`, and `Commit` get a fresh operation deadline: the earliest of the +network budget, statement timeout, and task-inherited request deadline. +Explicit and automatic rollback use an independent cleanup budget so an +expired operation budget cannot return a dirty connection to the pool. ODBC `SQL_ATTR_QUERY_TIMEOUT` has whole-second resolution. The driver rounds a positive sub-second value up when passing it to ODBC while retaining the exact @@ -64,6 +73,16 @@ The complete generated static-config schema, including the mutually exclusive `dsn`, `pools`, and `secdist_alias` connection sources, is available on components::Odbc. +All ODBC driver-manager and driver calls are synchronous and run on the task +processor selected by `blocking_task_processor`; if omitted, the global +blocking task processor is used. Production services may dedicate and size a +task processor for ODBC so slow driver calls do not contend with unrelated +blocking work. Exact sub-second cancellation is observed after a synchronous +driver call returns unless that driver implements its own timeout sooner. +At most five connection attempts per pool run concurrently. A temporary +startup outage does not discard successfully initialized connections or abort +the component; a background monitor retries until `min_pool_size` is restored. + For secdist, `odbc_settings.databases.` accepts either a `dsn` string or a `hosts` array. A host can be a DSN string or an object with a `dsn` member. Using secdist keeps credentials out of the static configuration and supports From ad867adeb69f57ac1a23b8ae61fd6a86a70c9cf6 Mon Sep 17 00:00:00 2001 From: Andrey Balabekyan <55596549+V0S7ER@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:35:05 +0300 Subject: [PATCH 03/12] feat odbc: add driver capability metadata --- odbc/src/storages/odbc/detail/connection.cpp | 30 +- odbc/src/storages/odbc/detail/connection.hpp | 6 + .../odbc/detail/driver_capabilities.cpp | 310 ++++++++++++++++++ .../odbc/detail/driver_capabilities.hpp | 103 ++++++ odbc/tests/odbc_postgresql_test.cpp | 40 +++ 5 files changed, 485 insertions(+), 4 deletions(-) create mode 100644 odbc/src/storages/odbc/detail/driver_capabilities.cpp create mode 100644 odbc/src/storages/odbc/detail/driver_capabilities.hpp diff --git a/odbc/src/storages/odbc/detail/connection.cpp b/odbc/src/storages/odbc/detail/connection.cpp index eabc9ed446f9..3543b2c927a7 100644 --- a/odbc/src/storages/odbc/detail/connection.cpp +++ b/odbc/src/storages/odbc/detail/connection.cpp @@ -41,6 +41,12 @@ namespace { using StatementHandle = std::unique_ptr, void (*)(SQLHSTMT)>; +struct ConnectedHandles final { + Connection::EnvironmentHandle environment; + Connection::DatabaseHandle database; + detail::DriverCapabilities driver_capabilities; +}; + template auto RunBlocking(engine::TaskProcessor& task_processor, Func&& func) -> std::invoke_result_t { const engine::TaskCancellationBlocker cancellation_blocker; @@ -572,21 +578,37 @@ Connection::Connection( throw MakeDriverError< ConnectionError>("Failed to connect to database", result, database.get(), SQL_HANDLE_DBC); } - return std::pair{std::move(environment), std::move(database)}; + if (deadline.IsReachable() && deadline.IsReached()) { + throw OperationInterrupted("Cancelled by deadline"); + } + + auto driver_capabilities = detail::DriverCapabilities::Read(database.get(), deadline); + return ConnectedHandles{ + std::move(environment), + std::move(database), + std::move(driver_capabilities), + }; }); try { CheckOperationInterrupted(deadline); } catch (...) { - DestroyConnectionHandlesOnBlockingTaskProcessor(blocking_task_processor_, handles.first, handles.second); + DestroyConnectionHandlesOnBlockingTaskProcessor( + blocking_task_processor_, + handles.environment, + handles.database + ); throw; } - env_ = std::move(handles.first); - handle_ = std::move(handles.second); + driver_capabilities_ = std::move(handles.driver_capabilities); + env_ = std::move(handles.environment); + handle_ = std::move(handles.database); } Connection::~Connection() { DestroyConnectionHandlesOnBlockingTaskProcessor(blocking_task_processor_, env_, handle_); } +const detail::DriverCapabilities& Connection::GetDriverCapabilities() const noexcept { return driver_capabilities_; } + ResultSet Connection::Query(std::string_view query) { return Query(query, impl::ParameterList{}, detail::GetExecuteDeadline(detail::kDefaultStatementTimeout)); } diff --git a/odbc/src/storages/odbc/detail/connection.hpp b/odbc/src/storages/odbc/detail/connection.hpp index 7f9a20115a35..9bbbdab7a4c4 100644 --- a/odbc/src/storages/odbc/detail/connection.hpp +++ b/odbc/src/storages/odbc/detail/connection.hpp @@ -16,6 +16,8 @@ #include #include +#include + USERVER_NAMESPACE_BEGIN namespace storages::odbc { @@ -65,6 +67,9 @@ class Connection final { detail::BrokenGuard GetBrokenGuard(); + /// Internal per-HDBC metadata snapshot captured during construction. + const detail::DriverCapabilities& GetDriverCapabilities() const noexcept; + private: friend class Transaction; void Begin(engine::Deadline deadline); @@ -78,6 +83,7 @@ class Connection final { mutable std::mutex handle_mutex_; EnvironmentHandle env_; DatabaseHandle handle_; + detail::DriverCapabilities driver_capabilities_; std::atomic broken_{false}; std::atomic in_transaction_{false}; }; diff --git a/odbc/src/storages/odbc/detail/driver_capabilities.cpp b/odbc/src/storages/odbc/detail/driver_capabilities.cpp new file mode 100644 index 000000000000..080246e14f0b --- /dev/null +++ b/odbc/src/storages/odbc/detail/driver_capabilities.cpp @@ -0,0 +1,310 @@ +#include + +#include +#include +#include +#include +#include + +#include + +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc::detail { + +namespace { + +constexpr std::size_t kInitialStringBufferSize = 256; +constexpr std::size_t kMaxWarningLength = 1024; + +enum class Requirement { kRequired, kOptional }; + +bool HasSqlState(const std::vector& diagnostics, std::string_view state) { + return std::any_of(diagnostics.begin(), diagnostics.end(), [state](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state == state; + }); +} + +bool IsUnsupportedInfo(const std::vector& diagnostics) { + return !diagnostics.empty() && + std::all_of(diagnostics.begin(), diagnostics.end(), [](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state == "HYC00" || diagnostic.sql_state == "HY096"; + }); +} + +void LogWarnings(std::string_view info_name, const std::vector& diagnostics) { + auto formatted = FormatSQLDiagnostics(diagnostics); + if (formatted.empty()) { + formatted = "no diagnostic records"; + } else if (formatted.size() > kMaxWarningLength) { + formatted.resize(kMaxWarningLength); + formatted += "..."; + } + LOG_WARNING() << "ODBC SQLGetInfo(" << info_name << ") completed with warning: " << formatted; +} + +[[noreturn]] void ThrowGetInfoError( + std::string_view info_name, + SQLRETURN result, + std::vector diagnostics +) { + auto message = fmt::format("Failed to read ODBC {} with SQLGetInfo", info_name); + const auto formatted = FormatSQLDiagnostics(diagnostics); + if (!formatted.empty()) { + message += ": "; + message += formatted; + } + throw ConnectionError{std::move(message), std::move(diagnostics), result == SQL_INVALID_HANDLE}; +} + +[[noreturn]] void ThrowInvalidInfoValue(std::string_view info_name, std::string_view value) { + throw ConnectionError{ + fmt::format("ODBC SQLGetInfo returned invalid {} value '{}'", info_name, value), + std::vector{}, + }; +} + +class InfoReader final { +public: + InfoReader(SQLHDBC connection, engine::Deadline deadline) + : connection_{connection}, + deadline_{deadline} + {} + + std::string ReadRequiredString(SQLUSMALLINT info_type, std::string_view info_name) const { + auto value = ReadString(info_type, info_name, Requirement::kRequired); + if (!value || value->empty()) { + ThrowInvalidInfoValue(info_name, value ? std::string_view{*value} : std::string_view{""}); + } + return std::move(*value); + } + + std::optional ReadOptionalYesNo(SQLUSMALLINT info_type, std::string_view info_name) const { + const auto value = ReadString(info_type, info_name, Requirement::kOptional); + if (!value) { + return std::nullopt; + } + if (*value == "Y") { + return true; + } + if (*value == "N") { + return false; + } + ThrowInvalidInfoValue(info_name, *value); + } + + template + std::optional ReadOptionalNumber(SQLUSMALLINT info_type, std::string_view info_name) const { + CheckDeadline(); + T value{}; + const auto + result = SQLGetInfo(connection_, info_type, &value, static_cast(sizeof(value)), nullptr); + CheckDeadline(); + auto diagnostics = + result == SQL_SUCCESS ? std::vector{} : GetSQLDiagnostics(connection_, SQL_HANDLE_DBC); + + if (!SQL_SUCCEEDED(result)) { + if (IsUnsupportedInfo(diagnostics)) { + return std::nullopt; + } + ThrowGetInfoError(info_name, result, std::move(diagnostics)); + } + if (HasSqlState(diagnostics, "01004")) { + // An exact-size numeric result cannot be usefully recovered by retrying. + ThrowGetInfoError(info_name, result, std::move(diagnostics)); + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogWarnings(info_name, diagnostics); + } + return value; + } + +private: + void CheckDeadline() const { + if (deadline_.IsReachable() && deadline_.IsReached()) { + throw OperationInterrupted("Cancelled by deadline"); + } + } + + std::optional ReadString(SQLUSMALLINT info_type, std::string_view info_name, Requirement requirement) + const { + constexpr auto kMaxBufferSize = static_cast(std::numeric_limits::max()); + std::vector buffer(kInitialStringBufferSize); + + while (true) { + CheckDeadline(); + SQLSMALLINT length = 0; + const auto result = + SQLGetInfo(connection_, info_type, buffer.data(), static_cast(buffer.size()), &length); + CheckDeadline(); + auto diagnostics = + result == SQL_SUCCESS + ? std::vector{} + : GetSQLDiagnostics(connection_, SQL_HANDLE_DBC); + + if (!SQL_SUCCEEDED(result)) { + if (requirement == Requirement::kOptional && IsUnsupportedInfo(diagnostics)) { + return std::nullopt; + } + ThrowGetInfoError(info_name, result, std::move(diagnostics)); + } + if (length < 0) { + ThrowGetInfoError(info_name, result, std::move(diagnostics)); + } + + const auto value_size = static_cast(length); + const bool truncated = HasSqlState(diagnostics, "01004") || value_size >= buffer.size(); + if (truncated) { + if (result == SQL_SUCCESS_WITH_INFO) { + auto non_truncation_diagnostics = diagnostics; + non_truncation_diagnostics.erase( + std::remove_if( + non_truncation_diagnostics.begin(), + non_truncation_diagnostics.end(), + [](const DiagnosticRecord& diagnostic) { return diagnostic.sql_state == "01004"; } + ), + non_truncation_diagnostics.end() + ); + if (!non_truncation_diagnostics.empty()) { + LogWarnings(info_name, non_truncation_diagnostics); + } + } + + const auto reported_buffer_size = value_size < kMaxBufferSize ? value_size + 1 : kMaxBufferSize; + const auto doubled_buffer_size = std::min(kMaxBufferSize, buffer.size() * 2); + const auto next_buffer_size = std::max(reported_buffer_size, doubled_buffer_size); + if (next_buffer_size <= buffer.size()) { + ThrowGetInfoError(info_name, result, std::move(diagnostics)); + } + buffer.resize(next_buffer_size); + continue; + } + + if (result == SQL_SUCCESS_WITH_INFO) { + LogWarnings(info_name, diagnostics); + } + return std::string{reinterpret_cast(buffer.data()), value_size}; + } + } + + SQLHDBC connection_; + engine::Deadline deadline_; +}; + +std::optional ReadTransactionCapability(const InfoReader& reader) { + const auto value = reader.ReadOptionalNumber(SQL_TXN_CAPABLE, "SQL_TXN_CAPABLE"); + if (!value) { + return std::nullopt; + } + switch (*value) { + case SQL_TC_NONE: + return TransactionCapability::kNone; + case SQL_TC_DML: + return TransactionCapability::kDml; + case SQL_TC_ALL: + return TransactionCapability::kAll; + case SQL_TC_DDL_COMMIT: + return TransactionCapability::kDdlCommit; + case SQL_TC_DDL_IGNORE: + return TransactionCapability::kDdlIgnore; + default: + ThrowInvalidInfoValue("SQL_TXN_CAPABLE", fmt::format("{}", *value)); + } +} + +std::optional ReadParameterArrayRowCounts(const InfoReader& reader) { + const auto value = reader.ReadOptionalNumber(SQL_PARAM_ARRAY_ROW_COUNTS, "SQL_PARAM_ARRAY_ROW_COUNTS"); + if (!value) { + return std::nullopt; + } + switch (*value) { + case SQL_PARC_BATCH: + return ParameterArrayRowCounts::kBatch; + case SQL_PARC_NO_BATCH: + return ParameterArrayRowCounts::kNoBatch; + default: + ThrowInvalidInfoValue("SQL_PARAM_ARRAY_ROW_COUNTS", fmt::format("{}", *value)); + } +} + +std::optional ReadParameterArraySelects(const InfoReader& reader) { + const auto value = reader.ReadOptionalNumber(SQL_PARAM_ARRAY_SELECTS, "SQL_PARAM_ARRAY_SELECTS"); + if (!value) { + return std::nullopt; + } + switch (*value) { + case SQL_PAS_BATCH: + return ParameterArraySelects::kBatch; + case SQL_PAS_NO_BATCH: + return ParameterArraySelects::kNoBatch; + case SQL_PAS_NO_SELECT: + return ParameterArraySelects::kNoSelect; + default: + ThrowInvalidInfoValue("SQL_PARAM_ARRAY_SELECTS", fmt::format("{}", *value)); + } +} + +std::optional ReadCursorBehavior( + const InfoReader& reader, + SQLUSMALLINT info_type, + std::string_view info_name +) { + const auto value = reader.ReadOptionalNumber(info_type, info_name); + if (!value) { + return std::nullopt; + } + switch (*value) { + case SQL_CB_DELETE: + return CursorBehavior::kDelete; + case SQL_CB_CLOSE: + return CursorBehavior::kClose; + case SQL_CB_PRESERVE: + return CursorBehavior::kPreserve; + default: + ThrowInvalidInfoValue(info_name, fmt::format("{}", *value)); + } +} + +} // namespace + +DriverCapabilities DriverCapabilities::Read(SQLHDBC connection, engine::Deadline deadline) { + const InfoReader reader{connection, deadline}; + DriverCapabilities capabilities; + + capabilities.dbms_name_ = reader.ReadRequiredString(SQL_DBMS_NAME, "SQL_DBMS_NAME"); + capabilities.dbms_version_ = reader.ReadRequiredString(SQL_DBMS_VER, "SQL_DBMS_VER"); + capabilities.driver_name_ = reader.ReadRequiredString(SQL_DRIVER_NAME, "SQL_DRIVER_NAME"); + capabilities.driver_version_ = reader.ReadRequiredString(SQL_DRIVER_VER, "SQL_DRIVER_VER"); + capabilities.driver_odbc_version_ = reader.ReadRequiredString(SQL_DRIVER_ODBC_VER, "SQL_DRIVER_ODBC_VER"); + + capabilities.transaction_capability_ = ReadTransactionCapability(reader); + capabilities.transaction_isolation_options_ = reader.ReadOptionalNumber< + SQLUINTEGER>(SQL_TXN_ISOLATION_OPTION, "SQL_TXN_ISOLATION_OPTION"); + capabilities.default_transaction_isolation_ = reader.ReadOptionalNumber< + SQLUINTEGER>(SQL_DEFAULT_TXN_ISOLATION, "SQL_DEFAULT_TXN_ISOLATION"); + capabilities + .data_source_read_only_ = reader.ReadOptionalYesNo(SQL_DATA_SOURCE_READ_ONLY, "SQL_DATA_SOURCE_READ_ONLY"); + capabilities.describe_parameter_ = reader.ReadOptionalYesNo(SQL_DESCRIBE_PARAMETER, "SQL_DESCRIBE_PARAMETER"); + + capabilities.parameter_array_row_counts_ = ReadParameterArrayRowCounts(reader); + capabilities.parameter_array_selects_ = ReadParameterArraySelects(reader); + capabilities.batch_row_count_ = reader.ReadOptionalNumber(SQL_BATCH_ROW_COUNT, "SQL_BATCH_ROW_COUNT"); + + capabilities.scroll_options_ = reader.ReadOptionalNumber(SQL_SCROLL_OPTIONS, "SQL_SCROLL_OPTIONS"); + capabilities + .getdata_extensions_ = reader.ReadOptionalNumber(SQL_GETDATA_EXTENSIONS, "SQL_GETDATA_EXTENSIONS"); + capabilities + .cursor_commit_behavior_ = ReadCursorBehavior(reader, SQL_CURSOR_COMMIT_BEHAVIOR, "SQL_CURSOR_COMMIT_BEHAVIOR"); + capabilities.cursor_rollback_behavior_ = + ReadCursorBehavior(reader, SQL_CURSOR_ROLLBACK_BEHAVIOR, "SQL_CURSOR_ROLLBACK_BEHAVIOR"); + + return capabilities; +} + +} // namespace storages::odbc::detail + +USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/detail/driver_capabilities.hpp b/odbc/src/storages/odbc/detail/driver_capabilities.hpp new file mode 100644 index 000000000000..9fde5874752c --- /dev/null +++ b/odbc/src/storages/odbc/detail/driver_capabilities.hpp @@ -0,0 +1,103 @@ +#pragma once + +#include +#include + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc::detail { + +enum class TransactionCapability : SQLUSMALLINT { + kNone = SQL_TC_NONE, + kDml = SQL_TC_DML, + kAll = SQL_TC_ALL, + kDdlCommit = SQL_TC_DDL_COMMIT, + kDdlIgnore = SQL_TC_DDL_IGNORE, +}; + +enum class ParameterArrayRowCounts : SQLUINTEGER { + kBatch = SQL_PARC_BATCH, + kNoBatch = SQL_PARC_NO_BATCH, +}; + +enum class ParameterArraySelects : SQLUINTEGER { + kBatch = SQL_PAS_BATCH, + kNoBatch = SQL_PAS_NO_BATCH, + kNoSelect = SQL_PAS_NO_SELECT, +}; + +enum class CursorBehavior : SQLUSMALLINT { + kDelete = SQL_CB_DELETE, + kClose = SQL_CB_CLOSE, + kPreserve = SQL_CB_PRESERVE, +}; + +/// Metadata reported by one physical ODBC connection immediately after connect. +/// +/// The snapshot deliberately preserves raw ODBC masks and distinct enum values: +/// their interpretation belongs to the operation that will eventually gate on +/// them. A missing optional value means that SQLGetInfo itself was unsupported. +class DriverCapabilities final { +public: + DriverCapabilities() = default; + + static DriverCapabilities Read(SQLHDBC connection, engine::Deadline deadline); + + const std::string& GetDbmsName() const noexcept { return dbms_name_; } + const std::string& GetDbmsVersion() const noexcept { return dbms_version_; } + const std::string& GetDriverName() const noexcept { return driver_name_; } + const std::string& GetDriverVersion() const noexcept { return driver_version_; } + const std::string& GetDriverOdbcVersion() const noexcept { return driver_odbc_version_; } + + std::optional GetTransactionCapability() const noexcept { return transaction_capability_; } + std::optional GetTransactionIsolationOptions() const noexcept { + return transaction_isolation_options_; + } + std::optional GetDefaultTransactionIsolation() const noexcept { + return default_transaction_isolation_; + } + std::optional IsDataSourceReadOnly() const noexcept { return data_source_read_only_; } + std::optional CanDescribeParameters() const noexcept { return describe_parameter_; } + + std::optional GetParameterArrayRowCounts() const noexcept { + return parameter_array_row_counts_; + } + std::optional GetParameterArraySelects() const noexcept { return parameter_array_selects_; } + std::optional GetBatchRowCount() const noexcept { return batch_row_count_; } + + std::optional GetScrollOptions() const noexcept { return scroll_options_; } + std::optional GetGetDataExtensions() const noexcept { return getdata_extensions_; } + std::optional GetCursorCommitBehavior() const noexcept { return cursor_commit_behavior_; } + std::optional GetCursorRollbackBehavior() const noexcept { return cursor_rollback_behavior_; } + +private: + std::string dbms_name_; + std::string dbms_version_; + std::string driver_name_; + std::string driver_version_; + std::string driver_odbc_version_; + + std::optional transaction_capability_; + std::optional transaction_isolation_options_; + std::optional default_transaction_isolation_; + std::optional data_source_read_only_; + std::optional describe_parameter_; + + std::optional parameter_array_row_counts_; + std::optional parameter_array_selects_; + std::optional batch_row_count_; + + std::optional scroll_options_; + std::optional getdata_extensions_; + std::optional cursor_commit_behavior_; + std::optional cursor_rollback_behavior_; +}; + +} // namespace storages::odbc::detail + +USERVER_NAMESPACE_END diff --git a/odbc/tests/odbc_postgresql_test.cpp b/odbc/tests/odbc_postgresql_test.cpp index 7e2702beb231..97e114042fd2 100644 --- a/odbc/tests/odbc_postgresql_test.cpp +++ b/odbc/tests/odbc_postgresql_test.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include #include @@ -18,6 +19,45 @@ UTEST(CreateConnection, Works) { auto cluster = MakeCluster(); } UTEST(CreateConnection, MultipleDSN) { auto cluster = MakeCluster(kMultiDSNSettings); } +UTEST(DriverCapabilities, CapturesPsqlOdbcSnapshot) { + Connection connection{kDSN}; + const auto& capabilities = connection.GetDriverCapabilities(); + + EXPECT_EQ(capabilities.GetDbmsName(), "PostgreSQL"); + EXPECT_FALSE(capabilities.GetDbmsVersion().empty()); + EXPECT_NE(capabilities.GetDriverName().find("psqlodbc"), std::string::npos); + EXPECT_FALSE(capabilities.GetDriverVersion().empty()); + EXPECT_FALSE(capabilities.GetDriverOdbcVersion().empty()); + + const auto transaction_capability = capabilities.GetTransactionCapability(); + ASSERT_TRUE(transaction_capability); + EXPECT_NE(*transaction_capability, detail::TransactionCapability::kNone); + + const auto isolation_options = capabilities.GetTransactionIsolationOptions(); + ASSERT_TRUE(isolation_options); + EXPECT_NE(*isolation_options & SQL_TXN_READ_COMMITTED, 0U); + const auto default_isolation = capabilities.GetDefaultTransactionIsolation(); + ASSERT_TRUE(default_isolation); + EXPECT_NE(*default_isolation, 0U); + EXPECT_EQ(*default_isolation & *isolation_options, *default_isolation); + + ASSERT_TRUE(capabilities.IsDataSourceReadOnly()); + EXPECT_FALSE(*capabilities.IsDataSourceReadOnly()); + ASSERT_TRUE(capabilities.CanDescribeParameters()); + EXPECT_FALSE(*capabilities.CanDescribeParameters()); + + EXPECT_TRUE(capabilities.GetParameterArrayRowCounts()); + EXPECT_TRUE(capabilities.GetParameterArraySelects()); + EXPECT_TRUE(capabilities.GetBatchRowCount()); + + const auto scroll_options = capabilities.GetScrollOptions(); + ASSERT_TRUE(scroll_options); + EXPECT_NE(*scroll_options & SQL_SO_FORWARD_ONLY, 0U); + EXPECT_TRUE(capabilities.GetGetDataExtensions()); + EXPECT_TRUE(capabilities.GetCursorCommitBehavior()); + EXPECT_TRUE(capabilities.GetCursorRollbackBehavior()); +} + UTEST(Query, Works) { auto cluster = MakeCluster(); From 077e006f78050f3b35c3baccd60552ab23ba61fa Mon Sep 17 00:00:00 2001 From: Andrey Balabekyan <55596549+V0S7ER@users.noreply.github.com> Date: Sun, 2 Aug 2026 03:54:26 +0300 Subject: [PATCH 04/12] feat odbc: add transaction options --- odbc/include/userver/storages/odbc.hpp | 1 + .../include/userver/storages/odbc/cluster.hpp | 10 + .../userver/storages/odbc/transaction.hpp | 8 + .../storages/odbc/transaction_options.hpp | 63 ++++ odbc/src/storages/odbc/cluster.cpp | 12 + .../src/storages/odbc/detail/cluster_impl.cpp | 16 +- .../src/storages/odbc/detail/cluster_impl.hpp | 8 + odbc/src/storages/odbc/detail/connection.cpp | 356 +++++++++++++++--- odbc/src/storages/odbc/detail/connection.hpp | 15 +- .../odbc/detail/transaction_options.cpp | 45 +++ .../odbc/detail/transaction_options.hpp | 25 ++ odbc/src/storages/odbc/transaction.cpp | 18 +- odbc/tests/odbc_transaction_test.cpp | 135 +++++++ scripts/docs/en/userver/odbc.md | 13 + 14 files changed, 666 insertions(+), 59 deletions(-) create mode 100644 odbc/include/userver/storages/odbc/transaction_options.hpp create mode 100644 odbc/src/storages/odbc/detail/transaction_options.cpp create mode 100644 odbc/src/storages/odbc/detail/transaction_options.hpp diff --git a/odbc/include/userver/storages/odbc.hpp b/odbc/include/userver/storages/odbc.hpp index 898db6f960e6..a060942608f9 100644 --- a/odbc/include/userver/storages/odbc.hpp +++ b/odbc/include/userver/storages/odbc.hpp @@ -8,6 +8,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN diff --git a/odbc/include/userver/storages/odbc/cluster.hpp b/odbc/include/userver/storages/odbc/cluster.hpp index 785a9d0b923e..694f63424349 100644 --- a/odbc/include/userver/storages/odbc/cluster.hpp +++ b/odbc/include/userver/storages/odbc/cluster.hpp @@ -65,6 +65,16 @@ class Cluster { Transaction Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control); + /// Start a transaction with explicit ODBC isolation/access options. + Transaction Begin(ClusterHostTypeFlags flags, const TransactionOptions& options); + + /// Start a transaction with explicit options and timeout overrides. + Transaction Begin( + ClusterHostTypeFlags flags, + const TransactionOptions& options, + OptionalCommandControl command_control + ); + void WriteStatistics(utils::statistics::Writer& writer) const; /// @brief Set default command control (timeouts) from dynamic config diff --git a/odbc/include/userver/storages/odbc/transaction.hpp b/odbc/include/userver/storages/odbc/transaction.hpp index 268ff068d9f4..afcfc9fe606f 100644 --- a/odbc/include/userver/storages/odbc/transaction.hpp +++ b/odbc/include/userver/storages/odbc/transaction.hpp @@ -14,6 +14,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN @@ -37,6 +38,13 @@ class Transaction final { std::chrono::milliseconds network_timeout, std::chrono::milliseconds statement_timeout ); + explicit Transaction( + detail::ConnectionPtr&& connection, + detail::Pool& pool, + const TransactionOptions& options, + std::chrono::milliseconds network_timeout, + std::chrono::milliseconds statement_timeout + ); ~Transaction(); Transaction(const Transaction& other) = delete; Transaction(Transaction&& other) noexcept; diff --git a/odbc/include/userver/storages/odbc/transaction_options.hpp b/odbc/include/userver/storages/odbc/transaction_options.hpp new file mode 100644 index 000000000000..5ae4c66cda78 --- /dev/null +++ b/odbc/include/userver/storages/odbc/transaction_options.hpp @@ -0,0 +1,63 @@ +#pragma once + +/// @file userver/storages/odbc/transaction_options.hpp +/// @brief ODBC transaction options + +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc { + +/// Portable ODBC transaction isolation levels. +enum class IsolationLevel : std::uint8_t { + kReadUncommitted, + kReadCommitted, + kRepeatableRead, + kSerializable, +}; + +/// ODBC transaction access-mode hint. +/// +/// @warning `kReadOnly` requests `SQL_MODE_READ_ONLY` from the ODBC driver, but +/// ODBC defines this as an intent/optimization hint. It does not guarantee that +/// the database rejects write statements. +enum class AccessMode : std::uint8_t { + kReadOnly, + kReadWrite, +}; + +/// Options for starting an ODBC transaction. +/// +/// Empty optionals preserve the physical connection's current driver defaults; +/// the driver does not silently force READ COMMITTED or READ WRITE. +struct TransactionOptions final { + std::optional isolation_level; + std::optional access_mode; + + // Explicit keeps the legacy Cluster::Begin(flags, {}) call unambiguous: an + // empty braced argument continues to mean OptionalCommandControl. + constexpr explicit TransactionOptions() = default; + + constexpr explicit TransactionOptions(IsolationLevel isolation) + : isolation_level{isolation} + {} + + constexpr explicit TransactionOptions(AccessMode mode) + : access_mode{mode} + {} + + constexpr TransactionOptions(IsolationLevel isolation, AccessMode mode) + : isolation_level{isolation}, + access_mode{mode} + {} +}; + +constexpr bool operator==(const TransactionOptions& lhs, const TransactionOptions& rhs) noexcept { + return lhs.isolation_level == rhs.isolation_level && lhs.access_mode == rhs.access_mode; +} + +} // namespace storages::odbc + +USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/cluster.cpp b/odbc/src/storages/odbc/cluster.cpp index de1f73235e2c..0fbaef6591a4 100644 --- a/odbc/src/storages/odbc/cluster.cpp +++ b/odbc/src/storages/odbc/cluster.cpp @@ -41,6 +41,18 @@ Transaction Cluster::Begin(ClusterHostTypeFlags flags, OptionalCommandControl co return impl_->Begin(flags, command_control); } +Transaction Cluster::Begin(ClusterHostTypeFlags flags, const TransactionOptions& options) { + return impl_->Begin(flags, options); +} + +Transaction Cluster::Begin( + ClusterHostTypeFlags flags, + const TransactionOptions& options, + OptionalCommandControl command_control +) { + return impl_->Begin(flags, options, command_control); +} + void Cluster::WriteStatistics(utils::statistics::Writer& writer) const { impl_->WriteStatistics(writer); } void Cluster::SetDefaultCommandControl(const CommandControl& cc) { impl_->SetDefaultCommandControl(cc); } diff --git a/odbc/src/storages/odbc/detail/cluster_impl.cpp b/odbc/src/storages/odbc/detail/cluster_impl.cpp index 488c56a50e55..315c70451844 100644 --- a/odbc/src/storages/odbc/detail/cluster_impl.cpp +++ b/odbc/src/storages/odbc/detail/cluster_impl.cpp @@ -84,9 +84,21 @@ ResultSet ClusterImpl::ExecuteImpl( } } -Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags) { return Begin(flags, std::nullopt); } +Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags) { return Begin(flags, TransactionOptions{}, std::nullopt); } Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control) { + return Begin(flags, TransactionOptions{}, command_control); +} + +Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags, const TransactionOptions& options) { + return Begin(flags, options, std::nullopt); +} + +Transaction ClusterImpl::Begin( + ClusterHostTypeFlags flags, + const TransactionOptions& options, + OptionalCommandControl command_control +) { const auto resolved = ResolveCommandControl(command_control); const auto network_timeout = resolved.network_timeout.value_or(kDefaultStatementTimeout); const auto statement_timeout = resolved.statement_timeout.value_or(kDefaultStatementTimeout); @@ -97,7 +109,7 @@ Transaction ClusterImpl::Begin(ClusterHostTypeFlags flags, OptionalCommandContro const auto topology = std::atomic_load(&topology_); auto& pool = SelectPool(*topology, flags); auto connection = pool.Acquire(acquire_deadline); - return Transaction{std::move(connection), pool, network_timeout, statement_timeout}; + return Transaction{std::move(connection), pool, options, network_timeout, statement_timeout}; } CommandControl ClusterImpl::ResolveCommandControl(OptionalCommandControl command_control) const { diff --git a/odbc/src/storages/odbc/detail/cluster_impl.hpp b/odbc/src/storages/odbc/detail/cluster_impl.hpp index 7de643b81aec..18f169b57a0a 100644 --- a/odbc/src/storages/odbc/detail/cluster_impl.hpp +++ b/odbc/src/storages/odbc/detail/cluster_impl.hpp @@ -49,6 +49,14 @@ class ClusterImpl { Transaction Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control); + Transaction Begin(ClusterHostTypeFlags flags, const TransactionOptions& options); + + Transaction Begin( + ClusterHostTypeFlags flags, + const TransactionOptions& options, + OptionalCommandControl command_control + ); + void WriteStatistics(utils::statistics::Writer& writer) const; void SetDefaultCommandControl(const CommandControl& cc); diff --git a/odbc/src/storages/odbc/detail/connection.cpp b/odbc/src/storages/odbc/detail/connection.cpp index 3543b2c927a7..6682dce377b5 100644 --- a/odbc/src/storages/odbc/detail/connection.cpp +++ b/odbc/src/storages/odbc/detail/connection.cpp @@ -31,6 +31,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -104,6 +105,130 @@ Exception MakeDriverError(std::string message, SQLRETURN result, SQLHANDLE handl return Exception{std::move(message), std::move(diagnostics), result == SQL_INVALID_HANDLE}; } +ConnectionError MakeConnectionError(std::string message, SQLRETURN result, std::vector diagnostics) { + const auto formatted = detail::FormatSQLDiagnostics(diagnostics); + if (!formatted.empty()) { + message += ": "; + message += formatted; + } + return ConnectionError{std::move(message), std::move(diagnostics), result == SQL_INVALID_HANDLE}; +} + +void LogConnectionAttributeWarnings(std::string_view operation, const std::vector& diagnostics) { + constexpr std::size_t kMaxWarningLength = 1024; + auto formatted = detail::FormatSQLDiagnostics(diagnostics); + if (formatted.size() > kMaxWarningLength) { + formatted.resize(kMaxWarningLength); + formatted += "..."; + } + LOG_WARNING() + << operation << " completed with warning: " << (formatted.empty() ? "no diagnostic records" : formatted); +} + +SQLUINTEGER ReadConnectionAttribute( + SQLHDBC connection, + SQLINTEGER attribute, + std::string_view attribute_name, + engine::Deadline deadline +) { + detail::CheckDeadlineNotExpired(deadline); + SQLUINTEGER value = 0; + SQLINTEGER value_size = 0; + const auto + result = SQLGetConnectAttr(connection, attribute, &value, static_cast(sizeof(value)), &value_size); + std::exception_ptr deadline_error; + try { + detail::CheckDeadlineNotExpired(deadline); + } catch (...) { + deadline_error = std::current_exception(); + } + + auto diagnostics = + result == SQL_SUCCESS ? std::vector{} : detail::GetSQLDiagnostics(connection, SQL_HANDLE_DBC); + if (!SQL_SUCCEEDED(result)) { + throw MakeConnectionError( + fmt::format("Failed to read ODBC connection attribute {}", attribute_name), + result, + std::move(diagnostics) + ); + } + if (std::any_of(diagnostics.begin(), diagnostics.end(), [](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state == "01004"; + })) + { + throw MakeConnectionError( + fmt::format("Failed to read exact ODBC connection attribute {}", attribute_name), + result, + std::move(diagnostics) + ); + } + if (deadline_error) { + std::rethrow_exception(deadline_error); + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogConnectionAttributeWarnings( + fmt::format("Reading ODBC connection attribute {}", attribute_name), + diagnostics + ); + } + return value; +} + +void SetConnectionAttributeVerified( + SQLHDBC connection, + SQLINTEGER attribute, + SQLUINTEGER requested_value, + std::string_view attribute_name, + engine::Deadline deadline +) { + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLSetConnectAttr( + connection, + attribute, + reinterpret_cast(static_cast(requested_value)), + SQL_IS_UINTEGER + ); + std::exception_ptr deadline_error; + try { + detail::CheckDeadlineNotExpired(deadline); + } catch (...) { + deadline_error = std::current_exception(); + } + + auto diagnostics = + result == SQL_SUCCESS ? std::vector{} : detail::GetSQLDiagnostics(connection, SQL_HANDLE_DBC); + if (!SQL_SUCCEEDED(result)) { + throw MakeConnectionError( + fmt::format("Failed to set requested ODBC connection attribute {}", attribute_name), + result, + std::move(diagnostics) + ); + } + if (deadline_error) { + std::rethrow_exception(deadline_error); + } + + const auto actual_value = ReadConnectionAttribute(connection, attribute, attribute_name, deadline); + if (!detail::IsExactConnectionAttributeValue(requested_value, actual_value)) { + throw MakeConnectionError( + fmt::format( + "ODBC driver substituted connection attribute {} value {} with {}", + attribute_name, + requested_value, + actual_value + ), + result, + std::move(diagnostics) + ); + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogConnectionAttributeWarnings( + fmt::format("Setting ODBC connection attribute {}", attribute_name), + diagnostics + ); + } +} + void DestroyEnvironmentHandle(SQLHENV handle) { if (handle != SQL_NULL_HENV) { SQLFreeHandle(SQL_HANDLE_ENV, handle); @@ -726,35 +851,101 @@ detail::BrokenGuard Connection::GetBrokenGuard() { return detail::BrokenGuard{*t bool Connection::IsInsideTransaction() const noexcept { return in_transaction_.load(); } -void Connection::Begin(engine::Deadline deadline) { - auto guard = GetBrokenGuard(); +void Connection::Begin(const TransactionOptions& options, engine::Deadline deadline) { + if (IsInsideTransaction()) { + throw ConnectionError("Cannot begin an ODBC transaction while another transaction is active"); + } + + const auto transaction_capability = driver_capabilities_.GetTransactionCapability(); + if (transaction_capability && *transaction_capability == detail::TransactionCapability::kNone) { + throw TransactionException("ODBC driver reports that transactions are not supported"); + } + if (options.isolation_level && + !detail::IsIsolationSupported(driver_capabilities_.GetTransactionIsolationOptions(), *options.isolation_level)) + { + throw TransactionException(fmt::format( + "ODBC driver does not report support for requested {} isolation", + detail::ToStringView(*options.isolation_level) + )); + } + try { - guard.Execute([this, deadline] { - RunBlockingChecked(blocking_task_processor_, deadline, [this, deadline] { - const std::lock_guard lock{handle_mutex_}; - detail::CheckDeadlineNotExpired(deadline); - const auto result = SQLSetConnectAttr( + RunBlockingChecked(blocking_task_processor_, deadline, [this, options, deadline] { + const std::lock_guard lock{handle_mutex_}; + TransactionAttributes snapshot; + if (options.isolation_level) { + snapshot.isolation = + ReadConnectionAttribute(handle_.get(), SQL_ATTR_TXN_ISOLATION, "SQL_ATTR_TXN_ISOLATION", deadline); + } + if (options.access_mode) { + snapshot.access_mode = + ReadConnectionAttribute(handle_.get(), SQL_ATTR_ACCESS_MODE, "SQL_ATTR_ACCESS_MODE", deadline); + } + snapshot.autocommit = + ReadConnectionAttribute(handle_.get(), SQL_ATTR_AUTOCOMMIT, "SQL_ATTR_AUTOCOMMIT", deadline); + + TransactionAttributes attempted_restore; + try { + if (options.isolation_level) { + attempted_restore.isolation = snapshot.isolation; + SetConnectionAttributeVerified( + handle_.get(), + SQL_ATTR_TXN_ISOLATION, + detail::ToOdbcIsolation(*options.isolation_level), + "SQL_ATTR_TXN_ISOLATION", + deadline + ); + } + if (options.access_mode) { + attempted_restore.access_mode = snapshot.access_mode; + const auto access_mode = + *options.access_mode == AccessMode::kReadOnly ? SQL_MODE_READ_ONLY : SQL_MODE_READ_WRITE; + SetConnectionAttributeVerified( + handle_.get(), + SQL_ATTR_ACCESS_MODE, + access_mode, + "SQL_ATTR_ACCESS_MODE", + deadline + ); + } + + attempted_restore.autocommit = snapshot.autocommit; + SetConnectionAttributeVerified( handle_.get(), SQL_ATTR_AUTOCOMMIT, - reinterpret_cast(SQL_AUTOCOMMIT_OFF), - SQL_IS_UINTEGER + SQL_AUTOCOMMIT_OFF, + "SQL_ATTR_AUTOCOMMIT", + deadline ); - if (!SQL_SUCCEEDED(result)) { - throw MakeDriverError( - "Failed to set connection autocommit attribute", - result, - handle_.get(), - SQL_HANDLE_DBC - ); + } catch (...) { + const auto begin_error = std::current_exception(); + if (attempted_restore.isolation || attempted_restore.access_mode || attempted_restore.autocommit) { + try { + RestoreTransactionAttributes( + attempted_restore, + engine::Deadline::FromDuration(detail::kDefaultCleanupTimeout) + ); + } catch (const std::exception& ex) { + NotifyBroken(); + LOG_ERROR() << "Failed to restore ODBC attributes after transaction begin failure: " << ex; + } catch (...) { + NotifyBroken(); + LOG_ERROR() << "Failed to restore ODBC attributes after transaction begin failure"; + } } - in_transaction_.store(true); - }); + std::rethrow_exception(begin_error); + } + + transaction_attributes_snapshot_ = snapshot; + in_transaction_.store(true); }); } catch (const OperationInterrupted&) { - // SQLSetConnectAttr may have completed before the caller observed the - // deadline/cancellation. Do not pool a connection with uncertain - // autocommit state. - NotifyBroken(); + CleanupInterruptedBegin(); + throw; + } catch (const ConnectionError& ex) { + if (ex.IsInvalidHandle() || ex.HasSqlStateClass("08")) { + NotifyBroken(); + } throw; } } @@ -768,17 +959,7 @@ void Connection::Commit(engine::Deadline deadline) { } RunBlockingChecked(blocking_task_processor_, deadline, [this, deadline] { const std::lock_guard lock{handle_mutex_}; - detail::CheckDeadlineNotExpired(deadline); - const auto result = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), SQL_COMMIT); - if (!SQL_SUCCEEDED(result)) { - throw MakeDriverError( - "Failed to commit transaction inside connection", - result, - handle_.get(), - SQL_HANDLE_DBC - ); - } - RestoreAutocommit(); + EndTransaction(SQL_COMMIT, "commit", deadline); }); }); } catch (const OperationInterrupted&) { @@ -796,17 +977,7 @@ void Connection::Rollback(engine::Deadline deadline) { } RunBlockingChecked(blocking_task_processor_, deadline, [this, deadline] { const std::lock_guard lock{handle_mutex_}; - detail::CheckDeadlineNotExpired(deadline); - const auto result = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), SQL_ROLLBACK); - if (!SQL_SUCCEEDED(result)) { - throw MakeDriverError( - "Failed to rollback transaction inside connection", - result, - handle_.get(), - SQL_HANDLE_DBC - ); - } - RestoreAutocommit(); + EndTransaction(SQL_ROLLBACK, "roll back", deadline); }); }); } catch (const OperationInterrupted&) { @@ -815,18 +986,95 @@ void Connection::Rollback(engine::Deadline deadline) { } } -void Connection::RestoreAutocommit() { - SQLRETURN ret = SQLSetConnectAttr( - handle_.get(), - SQL_ATTR_AUTOCOMMIT, - reinterpret_cast(SQL_AUTOCOMMIT_ON), - SQL_IS_UINTEGER - ); - if (!SQL_SUCCEEDED(ret)) { - throw MakeDriverError< - ConnectionError>("Failed to restore autocommit after transaction", ret, handle_.get(), SQL_HANDLE_DBC); +void Connection::EndTransaction(SQLSMALLINT completion_type, std::string_view operation, engine::Deadline deadline) { + if (!transaction_attributes_snapshot_) { + throw ConnectionError("ODBC transaction attribute snapshot is missing"); + } + + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLEndTran(SQL_HANDLE_DBC, handle_.get(), completion_type); + std::exception_ptr deadline_error; + try { + detail::CheckDeadlineNotExpired(deadline); + } catch (...) { + deadline_error = std::current_exception(); } + auto diagnostics = + result == SQL_SUCCESS + ? std::vector{} + : detail::GetSQLDiagnostics(handle_.get(), SQL_HANDLE_DBC); + if (!SQL_SUCCEEDED(result)) { + throw MakeConnectionError( + fmt::format("Failed to {} ODBC transaction", operation), + result, + std::move(diagnostics) + ); + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogConnectionAttributeWarnings(fmt::format("ODBC transaction {}", operation), diagnostics); + } + + RestoreTransactionAttributes( + *transaction_attributes_snapshot_, + engine::Deadline::FromDuration(detail::kDefaultCleanupTimeout) + ); + transaction_attributes_snapshot_.reset(); in_transaction_.store(false); + + if (deadline_error) { + std::rethrow_exception(deadline_error); + } +} + +void Connection::RestoreTransactionAttributes(const TransactionAttributes& attributes, engine::Deadline deadline) { + std::exception_ptr first_error; + const auto restore = [&](SQLINTEGER attribute, const std::optional& value, std::string_view name) { + if (!value) { + return; + } + try { + SetConnectionAttributeVerified(handle_.get(), attribute, *value, name, deadline); + } catch (...) { + if (!first_error) { + first_error = std::current_exception(); + } + } + }; + + // SQLEndTran leaves manual-commit mode active but no transaction open, so + // isolation and access mode are legal to restore before autocommit. + restore(SQL_ATTR_TXN_ISOLATION, attributes.isolation, "SQL_ATTR_TXN_ISOLATION"); + restore(SQL_ATTR_ACCESS_MODE, attributes.access_mode, "SQL_ATTR_ACCESS_MODE"); + restore(SQL_ATTR_AUTOCOMMIT, attributes.autocommit, "SQL_ATTR_AUTOCOMMIT"); + + if (first_error) { + std::rethrow_exception(first_error); + } +} + +void Connection::CleanupInterruptedBegin() noexcept { + if (!IsInsideTransaction()) { + return; + } + + try { + RunBlocking(blocking_task_processor_, [this] { + const std::lock_guard lock{handle_mutex_}; + if (IsInsideTransaction()) { + EndTransaction( + SQL_ROLLBACK, + "roll back interrupted begin", + engine::Deadline::FromDuration(detail::kDefaultCleanupTimeout) + ); + } + }); + } catch (const std::exception& ex) { + NotifyBroken(); + LOG_ERROR() << "Failed to clean up interrupted ODBC transaction begin: " << ex; + } catch (...) { + NotifyBroken(); + LOG_ERROR() << "Failed to clean up interrupted ODBC transaction begin"; + } } } // namespace storages::odbc diff --git a/odbc/src/storages/odbc/detail/connection.hpp b/odbc/src/storages/odbc/detail/connection.hpp index 9bbbdab7a4c4..4d6e1a355b43 100644 --- a/odbc/src/storages/odbc/detail/connection.hpp +++ b/odbc/src/storages/odbc/detail/connection.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include @@ -15,6 +16,7 @@ #include #include #include +#include #include @@ -71,19 +73,28 @@ class Connection final { const detail::DriverCapabilities& GetDriverCapabilities() const noexcept; private: + struct TransactionAttributes final { + std::optional isolation; + std::optional access_mode; + std::optional autocommit; + }; + friend class Transaction; - void Begin(engine::Deadline deadline); + void Begin(const TransactionOptions& options, engine::Deadline deadline); void Commit(engine::Deadline deadline); void Rollback(engine::Deadline deadline); bool IsInsideTransaction() const noexcept; - void RestoreAutocommit(); + void EndTransaction(SQLSMALLINT completion_type, std::string_view operation, engine::Deadline deadline); + void RestoreTransactionAttributes(const TransactionAttributes& attributes, engine::Deadline deadline); + void CleanupInterruptedBegin() noexcept; void UpdateBrokenFromDriver() noexcept; engine::TaskProcessor& blocking_task_processor_; mutable std::mutex handle_mutex_; EnvironmentHandle env_; DatabaseHandle handle_; detail::DriverCapabilities driver_capabilities_; + std::optional transaction_attributes_snapshot_; std::atomic broken_{false}; std::atomic in_transaction_{false}; }; diff --git a/odbc/src/storages/odbc/detail/transaction_options.cpp b/odbc/src/storages/odbc/detail/transaction_options.cpp new file mode 100644 index 000000000000..55bc80313a05 --- /dev/null +++ b/odbc/src/storages/odbc/detail/transaction_options.cpp @@ -0,0 +1,45 @@ +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc::detail { + +SQLUINTEGER ToOdbcIsolation(IsolationLevel isolation) noexcept { + switch (isolation) { + case IsolationLevel::kReadUncommitted: + return SQL_TXN_READ_UNCOMMITTED; + case IsolationLevel::kReadCommitted: + return SQL_TXN_READ_COMMITTED; + case IsolationLevel::kRepeatableRead: + return SQL_TXN_REPEATABLE_READ; + case IsolationLevel::kSerializable: + return SQL_TXN_SERIALIZABLE; + } + UINVARIANT(false, "Unknown ODBC isolation level"); +} + +std::string_view ToStringView(IsolationLevel isolation) noexcept { + switch (isolation) { + case IsolationLevel::kReadUncommitted: + return "READ UNCOMMITTED"; + case IsolationLevel::kReadCommitted: + return "READ COMMITTED"; + case IsolationLevel::kRepeatableRead: + return "REPEATABLE READ"; + case IsolationLevel::kSerializable: + return "SERIALIZABLE"; + } + UINVARIANT(false, "Unknown ODBC isolation level"); +} + +bool IsIsolationSupported(std::optional supported_mask, IsolationLevel isolation) noexcept { + return supported_mask && (*supported_mask & ToOdbcIsolation(isolation)) != 0; +} + +bool IsExactConnectionAttributeValue(SQLUINTEGER requested, SQLUINTEGER actual) noexcept { return requested == actual; } + +} // namespace storages::odbc::detail + +USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/detail/transaction_options.hpp b/odbc/src/storages/odbc/detail/transaction_options.hpp new file mode 100644 index 000000000000..e3cb7687e3a3 --- /dev/null +++ b/odbc/src/storages/odbc/detail/transaction_options.hpp @@ -0,0 +1,25 @@ +#pragma once + +#include +#include + +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc::detail { + +SQLUINTEGER ToOdbcIsolation(IsolationLevel isolation) noexcept; + +std::string_view ToStringView(IsolationLevel isolation) noexcept; + +bool IsIsolationSupported(std::optional supported_mask, IsolationLevel isolation) noexcept; + +bool IsExactConnectionAttributeValue(SQLUINTEGER requested, SQLUINTEGER actual) noexcept; + +} // namespace storages::odbc::detail + +USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/transaction.cpp b/odbc/src/storages/odbc/transaction.cpp index 7a67f162972a..767eff0437d8 100644 --- a/odbc/src/storages/odbc/transaction.cpp +++ b/odbc/src/storages/odbc/transaction.cpp @@ -23,6 +23,22 @@ Transaction::Transaction( detail::Pool& pool, std::chrono::milliseconds network_timeout, std::chrono::milliseconds statement_timeout +) + : Transaction{ + std::move(connection), + pool, + TransactionOptions{}, + network_timeout, + statement_timeout, + } +{} + +Transaction::Transaction( + detail::ConnectionPtr&& connection, + detail::Pool& pool, + const TransactionOptions& options, + std::chrono::milliseconds network_timeout, + std::chrono::milliseconds statement_timeout ) : connection_{std::move(connection)}, pool_{&pool}, @@ -35,7 +51,7 @@ Transaction::Transaction( const auto deadline = std::min(detail::GetExecuteDeadline(network_timeout_), detail::GetExecuteDeadline(statement_timeout_)); detail::CheckDeadlineNotExpired(deadline); - (*connection_)->Begin(deadline); + (*connection_)->Begin(options, deadline); trx_lock_.Lock(); pool_->AccountTransactionStarted(); } diff --git a/odbc/tests/odbc_transaction_test.cpp b/odbc/tests/odbc_transaction_test.cpp index dd2b08070e53..29e4d5162ef4 100644 --- a/odbc/tests/odbc_transaction_test.cpp +++ b/odbc/tests/odbc_transaction_test.cpp @@ -1,4 +1,6 @@ #include +#include +#include #include #include #include @@ -12,6 +14,31 @@ namespace storages::odbc::tests { using namespace std::chrono_literals; +static_assert(requires(Cluster& cluster, CommandControl command_control, TransactionOptions options) { + cluster.Begin(ClusterHostType::kMaster); + cluster.Begin(ClusterHostType::kMaster, {}); + cluster.Begin(ClusterHostType::kMaster, std::nullopt); + cluster.Begin(ClusterHostType::kMaster, command_control); + cluster.Begin(ClusterHostType::kMaster, options); + cluster.Begin(ClusterHostType::kMaster, options, {}); + cluster.Begin(ClusterHostType::kMaster, options, std::nullopt); + cluster.Begin(ClusterHostType::kMaster, options, command_control); + cluster.Begin(ClusterHostType::kMaster, TransactionOptions{IsolationLevel::kSerializable}); + cluster.Begin(ClusterHostType::kMaster, TransactionOptions{AccessMode::kReadOnly}); +}); + +void ExerciseReadOnlyRequest(Cluster& cluster) { + try { + auto read_only = cluster.Begin( + ClusterHostType::kMaster, + TransactionOptions{IsolationLevel::kSerializable, AccessMode::kReadOnly} + ); + read_only.Rollback(); + } catch (const ConnectionError& ex) { + EXPECT_NE(std::string_view{ex.what()}.find("SQL_ATTR_ACCESS_MODE"), std::string_view::npos); + } +} + UTEST(CreateTransaction, Works) { auto cluster = MakeCluster(); auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); @@ -30,6 +57,114 @@ UTEST(CreateTransaction, Rollback) { trx.Rollback(); } +UTEST(TransactionOptions, IsolationMaskPreflight) { + EXPECT_FALSE(detail::IsIsolationSupported(std::nullopt, IsolationLevel::kSerializable)); + EXPECT_FALSE(detail::IsIsolationSupported(SQL_TXN_READ_COMMITTED, IsolationLevel::kSerializable)); + EXPECT_TRUE(detail::IsIsolationSupported(SQL_TXN_SERIALIZABLE, IsolationLevel::kSerializable)); + EXPECT_TRUE( + detail::IsIsolationSupported(SQL_TXN_READ_COMMITTED | SQL_TXN_SERIALIZABLE, IsolationLevel::kReadCommitted) + ); + EXPECT_TRUE(detail::IsExactConnectionAttributeValue(SQL_MODE_READ_ONLY, SQL_MODE_READ_ONLY)); + EXPECT_FALSE(detail::IsExactConnectionAttributeValue(SQL_MODE_READ_ONLY, SQL_MODE_READ_WRITE)); +} + +UTEST(TransactionOptions, AppliesSupportedIsolationLevelsExactly) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + + const std::array cases{ + std::pair{IsolationLevel::kReadUncommitted, "read uncommitted"}, + std::pair{IsolationLevel::kReadCommitted, "read committed"}, + std::pair{IsolationLevel::kRepeatableRead, "repeatable read"}, + std::pair{IsolationLevel::kSerializable, "serializable"}, + }; + for (const auto& [isolation, expected] : cases) { + auto trx = cluster.Begin(ClusterHostType::kMaster, TransactionOptions{isolation}); + const auto result = trx.Execute("SHOW transaction_isolation"); + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetString(), expected); + trx.Commit(); + } +} + +UTEST(TransactionOptions, ReadOnlyNeverSilentlyDowngradesAndConnectionIsReusable) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + cluster.Execute(ClusterHostType::kMaster, "CREATE TEMP TABLE odbc_transaction_options(value INTEGER)"); + + /// [ODBC transaction options] + const TransactionOptions read_write_serializable{ + IsolationLevel::kSerializable, + AccessMode::kReadWrite, + }; + auto configured = cluster.Begin(ClusterHostType::kMaster, read_write_serializable); + const auto result = configured.Execute("SELECT 1"); + ASSERT_EQ(result.Size(), 1); + configured.Rollback(); + /// [ODBC transaction options] + + // Drivers may either apply SQL_MODE_READ_ONLY exactly or reject/substitute + // it. Both are valid; continuing silently with READ_WRITE is not. + ExerciseReadOnlyRequest(cluster); + + auto read_write = cluster.Begin(ClusterHostType::kMaster, TransactionOptions{AccessMode::kReadWrite}); + read_write.Execute("INSERT INTO odbc_transaction_options VALUES (1)"); + read_write.Commit(); + + auto defaults = cluster.Begin(ClusterHostType::kMaster, TransactionOptions{}); + defaults.Execute("INSERT INTO odbc_transaction_options VALUES (2)"); + defaults.Commit(); + + const auto count = cluster.Execute(ClusterHostType::kMaster, "SELECT COUNT(*) FROM odbc_transaction_options"); + ASSERT_EQ(count.Size(), 1); + EXPECT_EQ(count[0][0].GetInt64(), 2); +} + +UTEST(TransactionOptions, RestoresConnectionAttributesAcrossAllExitPaths) { + const auto host_settings = storages::odbc::settings::HostSettings{kDSN, {1, 1}}; + storages::odbc::Cluster cluster(storages::odbc::settings::ODBCClusterSettings{{host_settings}}, nullptr); + + const auto expect_default = [&cluster] { + auto trx = cluster.Begin(ClusterHostType::kMaster, TransactionOptions{}); + const auto result = trx.Execute("SHOW transaction_isolation"); + EXPECT_EQ(result[0][0].GetString(), "read committed"); + trx.Commit(); + }; + + ExerciseReadOnlyRequest(cluster); + expect_default(); + + { + auto trx = cluster.Begin( + ClusterHostType::kMaster, + TransactionOptions{IsolationLevel::kSerializable, AccessMode::kReadWrite} + ); + EXPECT_EQ(trx.Execute("SHOW transaction_isolation")[0][0].GetString(), "serializable"); + trx.Commit(); + } + expect_default(); + + { + auto trx = cluster.Begin( + ClusterHostType::kMaster, + TransactionOptions{IsolationLevel::kRepeatableRead, AccessMode::kReadWrite} + ); + EXPECT_EQ(trx.Execute("SHOW transaction_isolation")[0][0].GetString(), "repeatable read"); + trx.Rollback(); + } + expect_default(); + + { + auto trx = cluster.Begin( + ClusterHostType::kMaster, + TransactionOptions{IsolationLevel::kSerializable, AccessMode::kReadWrite} + ); + EXPECT_EQ(trx.Execute("SHOW transaction_isolation")[0][0].GetString(), "serializable"); + // RAII rollback must restore all connection attributes for the next borrower. + } + expect_default(); +} + UTEST(Transaction, QueryInTransaction) { auto cluster = MakeCluster(); auto trx = cluster.Begin(storages::odbc::ClusterHostType::kMaster); diff --git a/scripts/docs/en/userver/odbc.md b/scripts/docs/en/userver/odbc.md index e6bb985caf09..a81ae57f05dc 100644 --- a/scripts/docs/en/userver/odbc.md +++ b/scripts/docs/en/userver/odbc.md @@ -41,6 +41,19 @@ Parameters are bound in transaction queries in exactly the same way: @snippet odbc/tests/odbc_transaction_test.cpp ODBC transaction parameter binding +storages::odbc::TransactionOptions can request any portable ODBC isolation +level and `SQL_ATTR_ACCESS_MODE` read-only/read-write hints: + +@snippet odbc/tests/odbc_transaction_test.cpp ODBC transaction options + +Default-constructed options do not override either connection attribute, so +the selected driver's current defaults are preserved. Explicit isolation is +accepted only when the physical connection reports the requested level and +then applies it exactly; the driver never silently substitutes a weaker level. +ODBC defines read-only access mode as an intent/optimization hint. Applications +must not rely on it as authorization or assume that write statements will be +rejected by every driver/database combination. + ### Command control and deadlines storages::odbc::CommandControl configures the connection-acquisition/network From c2605dd33949706146ad19f82c31fce3203a472b Mon Sep 17 00:00:00 2001 From: Andrey Balabekyan <55596549+V0S7ER@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:06:42 +0300 Subject: [PATCH 05/12] feat odbc: add dynamic parameter store --- odbc/include/userver/storages/odbc.hpp | 1 + .../include/userver/storages/odbc/cluster.hpp | 12 ++ .../userver/storages/odbc/parameter_store.hpp | 98 ++++++++++++++ .../userver/storages/odbc/transaction.hpp | 7 + odbc/src/storages/odbc/cluster.cpp | 13 ++ odbc/src/storages/odbc/transaction.cpp | 12 ++ odbc/tests/odbc_postgresql_test.cpp | 121 ++++++++++++++++++ scripts/docs/en/userver/odbc.md | 13 ++ 8 files changed, 277 insertions(+) create mode 100644 odbc/include/userver/storages/odbc/parameter_store.hpp diff --git a/odbc/include/userver/storages/odbc.hpp b/odbc/include/userver/storages/odbc.hpp index a060942608f9..ea2aabbe8826 100644 --- a/odbc/include/userver/storages/odbc.hpp +++ b/odbc/include/userver/storages/odbc.hpp @@ -8,6 +8,7 @@ #include #include #include +#include #include USERVER_NAMESPACE_BEGIN diff --git a/odbc/include/userver/storages/odbc/cluster.hpp b/odbc/include/userver/storages/odbc/cluster.hpp index 694f63424349..9ac6e53d7671 100644 --- a/odbc/include/userver/storages/odbc/cluster.hpp +++ b/odbc/include/userver/storages/odbc/cluster.hpp @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -61,6 +62,17 @@ class Cluster { return DoExecute(command_control, flags, query, impl::MakeParameterList(args...)); } + /// @brief Execute a statement with an owning dynamic parameter list. + ResultSet Execute(ClusterHostTypeFlags flags, const Query& query, const ParameterStore& store); + + /// @brief Execute a statement with a dynamic parameter list and timeout overrides. + ResultSet Execute( + ClusterHostTypeFlags flags, + OptionalCommandControl command_control, + const Query& query, + const ParameterStore& store + ); + Transaction Begin(ClusterHostTypeFlags flags); Transaction Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control); diff --git a/odbc/include/userver/storages/odbc/parameter_store.hpp b/odbc/include/userver/storages/odbc/parameter_store.hpp new file mode 100644 index 000000000000..36ab46245326 --- /dev/null +++ b/odbc/include/userver/storages/odbc/parameter_store.hpp @@ -0,0 +1,98 @@ +#pragma once + +/// @file userver/storages/odbc/parameter_store.hpp +/// @brief @copybrief storages::odbc::ParameterStore + +#include +#include +#include +#include +#include +#include + +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc { + +class Cluster; +class Transaction; + +/// @cond +namespace impl { + +template +struct IsParameterStoreValue { +private: + using Value = std::remove_cv_t; + using Pointee = std::remove_pointer_t; + using Element = std::remove_extent_t; + +public: + static constexpr bool value = + std::integral || std::floating_point || std::is_enum_v || + std::same_as || std::same_as || + std::same_as || std::same_as || + (std::is_pointer_v && (std::same_as || std::same_as)) || + (std::is_array_v && std::same_as, char>); +}; + +template +struct IsParameterStoreValue> final : IsParameterStoreValue> {}; + +template +inline constexpr bool kIsParameterStoreValue = IsParameterStoreValue>::value; + +} // namespace impl +/// @endcond + +/// @ingroup userver_containers +/// +/// @brief Owning, ordered list of dynamically assembled ODBC parameters. +/// +/// Values are copied into the store and remain valid independently of the +/// source objects. Use an empty `std::optional` for SQL NULL: `T` determines +/// the parameter type used for ODBC binding. Raw `nullptr` and `std::nullopt` +/// remain untyped, just like in the variadic API, and should only be used when +/// the driver can infer the type from the statement. A null `const char*` is a +/// typed string NULL. +/// +/// @warning Parameters are always values for existing `?` placeholders. Never +/// interpolate them into the SQL query text. +class ParameterStore final { +public: + ParameterStore() = default; + ParameterStore(const ParameterStore&) = delete; + ParameterStore(ParameterStore&&) noexcept = default; + ParameterStore& operator=(const ParameterStore&) = delete; + ParameterStore& operator=(ParameterStore&&) noexcept = default; + + /// @brief Copies a parameter supported by the variadic ODBC API to the end + /// of the ordered list. + /// @returns `*this` for chained construction. + template + requires(impl::kIsParameterStoreValue && std::constructible_from) + ParameterStore& PushBack(const T& parameter) { + parameters_.emplace_back(parameter); + return *this; + } + + /// Returns whether the parameter list is empty. + bool IsEmpty() const noexcept { return parameters_.empty(); } + + /// Returns the number of stored parameters. + std::size_t Size() const noexcept { return parameters_.size(); } + +private: + friend class Cluster; + friend class Transaction; + + const impl::ParameterList& GetParameters() const noexcept { return parameters_; } + + impl::ParameterList parameters_; +}; + +} // namespace storages::odbc + +USERVER_NAMESPACE_END diff --git a/odbc/include/userver/storages/odbc/transaction.hpp b/odbc/include/userver/storages/odbc/transaction.hpp index afcfc9fe606f..74c732c00bed 100644 --- a/odbc/include/userver/storages/odbc/transaction.hpp +++ b/odbc/include/userver/storages/odbc/transaction.hpp @@ -12,6 +12,7 @@ #include #include +#include #include #include #include @@ -61,6 +62,12 @@ class Transaction final { return DoExecute(command_control, query, impl::MakeParameterList(args...)); } + /// @brief Execute a statement with an owning dynamic parameter list. + ResultSet Execute(const Query& query, const ParameterStore& store); + + /// @brief Execute a statement with a dynamic parameter list and timeout overrides. + ResultSet Execute(OptionalCommandControl command_control, const Query& query, const ParameterStore& store); + /// @brief Commit the transaction void Commit(); diff --git a/odbc/src/storages/odbc/cluster.cpp b/odbc/src/storages/odbc/cluster.cpp index 0fbaef6591a4..13a63c898a08 100644 --- a/odbc/src/storages/odbc/cluster.cpp +++ b/odbc/src/storages/odbc/cluster.cpp @@ -35,6 +35,19 @@ ResultSet Cluster::DoExecute( return impl_->Execute(flags, command_control, query, parameters); } +ResultSet Cluster::Execute(ClusterHostTypeFlags flags, const Query& query, const ParameterStore& store) { + return Execute(flags, std::nullopt, query, store); +} + +ResultSet Cluster::Execute( + ClusterHostTypeFlags flags, + OptionalCommandControl command_control, + const Query& query, + const ParameterStore& store +) { + return DoExecute(command_control, flags, query, store.GetParameters()); +} + Transaction Cluster::Begin(ClusterHostTypeFlags flags) { return impl_->Begin(flags); } Transaction Cluster::Begin(ClusterHostTypeFlags flags, OptionalCommandControl command_control) { diff --git a/odbc/src/storages/odbc/transaction.cpp b/odbc/src/storages/odbc/transaction.cpp index 767eff0437d8..3de4fe265e71 100644 --- a/odbc/src/storages/odbc/transaction.cpp +++ b/odbc/src/storages/odbc/transaction.cpp @@ -75,6 +75,18 @@ Transaction::~Transaction() { } } +ResultSet Transaction::Execute(const Query& query, const ParameterStore& store) { + return Execute(std::nullopt, query, store); +} + +ResultSet Transaction::Execute( + OptionalCommandControl command_control, + const Query& query, + const ParameterStore& store +) { + return DoExecute(command_control, query, store.GetParameters()); +} + void Transaction::Commit() { const utils::FastScopeGuard unlock_guard([this]() noexcept { trx_lock_.Unlock(); }); AssertValid(); diff --git a/odbc/tests/odbc_postgresql_test.cpp b/odbc/tests/odbc_postgresql_test.cpp index 97e114042fd2..7e646022a89d 100644 --- a/odbc/tests/odbc_postgresql_test.cpp +++ b/odbc/tests/odbc_postgresql_test.cpp @@ -1,13 +1,17 @@ #include +#include #include #include #include #include #include +#include #include +#include #include #include #include +#include #include @@ -15,6 +19,43 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc::tests { +using ClusterParameterStoreExecute = ResultSet (Cluster::*)(ClusterHostTypeFlags, const Query&, const ParameterStore&); +using ClusterParameterStoreExecuteWithCommandControl = + ResultSet (Cluster::*)(ClusterHostTypeFlags, OptionalCommandControl, const Query&, const ParameterStore&); +using TransactionParameterStoreExecute = ResultSet (Transaction::*)(const Query&, const ParameterStore&); +using TransactionParameterStoreExecuteWithCommandControl = + ResultSet (Transaction::*)(OptionalCommandControl, const Query&, const ParameterStore&); + +template +concept ParameterStorePushable = requires(ParameterStore& store, const T& value) { store.PushBack(value); }; + +struct UnsupportedParameter final {}; + +static_assert(!std::is_copy_constructible_v); +static_assert(!std::is_copy_assignable_v); +static_assert(std::is_nothrow_move_constructible_v); +static_assert(std::is_nothrow_move_assignable_v); +static_assert(requires(ParameterStore& store, const std::optional& value) { + { + store.PushBack(value) + } -> std::same_as; + store.PushBack("string literal"); +}); +static_assert(ParameterStorePushable); +static_assert(ParameterStorePushable); +static_assert(ParameterStorePushable); +static_assert(ParameterStorePushable); +static_assert(ParameterStorePushable>); +static_assert(!ParameterStorePushable); +static_assert(!ParameterStorePushable); +static_assert(!ParameterStorePushable>>); +static_assert(requires { + static_cast(&Cluster::Execute); + static_cast(&Cluster::Execute); + static_cast(&Transaction::Execute); + static_cast(&Transaction::Execute); +}); + UTEST(CreateConnection, Works) { auto cluster = MakeCluster(); } UTEST(CreateConnection, MultipleDSN) { auto cluster = MakeCluster(kMultiDSNSettings); } @@ -107,6 +148,86 @@ UTEST(Query, BindsParametersWithoutInterpolation) { EXPECT_FALSE(result[0][6].GetBool()); } +UTEST(ParameterStore, OwnsDynamicParametersAndIsReusable) { + auto cluster = MakeCluster(); + + ParameterStore empty; + EXPECT_TRUE(empty.IsEmpty()); + EXPECT_EQ(empty.Size(), 0); + const auto empty_result = cluster.Execute(ClusterHostType::kMaster, "SELECT 1", empty); + ASSERT_EQ(empty_result.Size(), 1); + EXPECT_EQ(empty_result[0][0].GetInt32(), 1); + + /// [ODBC dynamic parameter store] + const std::string injection_payload = "Robert'); DROP TABLE users;--"; + std::string copied_source = injection_payload; + const std::optional null_integer; + const std::optional null_string; + + ParameterStore parameters; + parameters.PushBack(std::int64_t{42}) + .PushBack(copied_source) + .PushBack("string literal") + .PushBack(std::string{"temporary string"}) + .PushBack(null_integer) + .PushBack(null_string); + + const Query query{ + "SELECT ?::bigint, ?::text, ?::text, ?::text, ?::integer IS NULL, ?::text IS NULL", + }; + const auto result = cluster.Execute(ClusterHostType::kMaster, query, parameters); + /// [ODBC dynamic parameter store] + + copied_source.assign("changed after PushBack"); + EXPECT_FALSE(parameters.IsEmpty()); + EXPECT_EQ(parameters.Size(), 6); + + ParameterStore moved_parameters{std::move(parameters)}; + EXPECT_EQ(moved_parameters.Size(), 6); + + const auto validate = [&injection_payload](const ResultSet& value) { + ASSERT_EQ(value.Size(), 1); + EXPECT_EQ(value[0][0].GetInt64(), 42); + EXPECT_EQ(value[0][1].GetString(), injection_payload); + EXPECT_EQ(value[0][2].GetString(), "string literal"); + EXPECT_EQ(value[0][3].GetString(), "temporary string"); + EXPECT_TRUE(value[0][4].GetBool()); + EXPECT_TRUE(value[0][5].GetBool()); + }; + validate(result); + + // Reusing a store never consumes or mutates its values. + validate(cluster.Execute(ClusterHostType::kMaster, CommandControl{}, query, moved_parameters)); + + auto transaction = cluster.Begin(ClusterHostType::kMaster); + validate(transaction.Execute(query, moved_parameters)); + validate(transaction.Execute(CommandControl{}, query, moved_parameters)); + UEXPECT_THROW(transaction.Execute("SELECT ?::integer", moved_parameters), StatementError); + transaction.Rollback(); + + // Unlike raw nullptr, a null const char* retains the string parameter type. + const char* null_c_string = nullptr; + ParameterStore null_string_parameter; + null_string_parameter.PushBack(null_c_string); + const auto null_result = cluster.Execute(ClusterHostType::kMaster, "SELECT ?::text IS NULL", null_string_parameter); + ASSERT_EQ(null_result.Size(), 1); + EXPECT_TRUE(null_result[0][0].GetBool()); +} + +UTEST(ParameterStore, PreservesUnsignedBigintRangeChecks) { + auto cluster = MakeCluster(); + + ParameterStore supported; + supported.PushBack(static_cast(std::numeric_limits::max())); + const auto result = cluster.Execute(ClusterHostType::kMaster, "SELECT ?::bigint", supported); + ASSERT_EQ(result.Size(), 1); + EXPECT_EQ(result[0][0].GetInt64(), std::numeric_limits::max()); + + ParameterStore unsupported; + unsupported.PushBack(std::numeric_limits::max()); + UEXPECT_THROW(cluster.Execute(ClusterHostType::kMaster, "SELECT ?::numeric", unsupported), StatementError); +} + UTEST(Query, RejectsUnsignedValuesOutsidePortableBigintRange) { auto cluster = MakeCluster(); diff --git a/scripts/docs/en/userver/odbc.md b/scripts/docs/en/userver/odbc.md index a81ae57f05dc..0a233a4200f2 100644 --- a/scripts/docs/en/userver/odbc.md +++ b/scripts/docs/en/userver/odbc.md @@ -24,6 +24,19 @@ value, or `nullptr` when the ODBC driver can infer the parameter type from the statement. The number of C++ arguments must match the number of `?` placeholders. +For queries assembled at runtime, storages::odbc::ParameterStore provides an +owning, ordered dynamic parameter list with the same supported value types and +the same safe ODBC binding path: + +@snippet odbc/tests/odbc_postgresql_test.cpp ODBC dynamic parameter store + +`PushBack` copies each value, so the store can outlive source objects and can be +reused by cluster and transaction executions without being consumed. Prefer an +empty `std::optional` for NULL because `T` preserves the concrete binding +type. Raw `nullptr` and `std::nullopt` remain untyped and require driver type +inference; a null value whose static type is `const char*` is instead bound as +a typed string NULL. + `storages::odbc::Cluster::Execute` returns a storages::odbc::ResultSet. Its rows contain storages::odbc::Field values that provide typed getters such as `GetInt32`, `GetInt64`, `GetDouble`, `GetBool`, and `GetString`. From 57b19f97bf496b96f000e1de8b0b59793f675eb1 Mon Sep 17 00:00:00 2001 From: Andrey Balabekyan <55596549+V0S7ER@users.noreply.github.com> Date: Sun, 2 Aug 2026 04:54:47 +0300 Subject: [PATCH 06/12] feat odbc: add typed result mapping --- odbc/include/userver/storages/odbc.hpp | 1 + odbc/include/userver/storages/odbc/field.hpp | 113 +++ .../userver/storages/odbc/impl/parameter.hpp | 54 +- .../userver/storages/odbc/parameter_store.hpp | 3 +- .../userver/storages/odbc/result_set.hpp | 143 ++++ odbc/include/userver/storages/odbc/types.hpp | 251 +++++++ odbc/src/storages/odbc/detail/connection.cpp | 647 +++++++++++++++++- .../src/storages/odbc/detail/result_chunk.cpp | 61 ++ .../src/storages/odbc/detail/result_chunk.hpp | 36 + .../storages/odbc/detail/result_wrapper.cpp | 221 +++++- .../storages/odbc/detail/result_wrapper.hpp | 30 +- odbc/src/storages/odbc/field.cpp | 24 + odbc/src/storages/odbc/types.cpp | 125 ++++ odbc/tests/odbc_postgresql_test.cpp | 127 ++++ odbc/tests/odbc_types_test.cpp | 308 +++++++++ scripts/docs/en/userver/odbc.md | 59 +- 16 files changed, 2149 insertions(+), 54 deletions(-) create mode 100644 odbc/include/userver/storages/odbc/types.hpp create mode 100644 odbc/src/storages/odbc/detail/result_chunk.cpp create mode 100644 odbc/src/storages/odbc/detail/result_chunk.hpp create mode 100644 odbc/src/storages/odbc/types.cpp create mode 100644 odbc/tests/odbc_types_test.cpp diff --git a/odbc/include/userver/storages/odbc.hpp b/odbc/include/userver/storages/odbc.hpp index ea2aabbe8826..eda74695659d 100644 --- a/odbc/include/userver/storages/odbc.hpp +++ b/odbc/include/userver/storages/odbc.hpp @@ -10,6 +10,7 @@ #include #include #include +#include USERVER_NAMESPACE_BEGIN diff --git a/odbc/include/userver/storages/odbc/field.hpp b/odbc/include/userver/storages/odbc/field.hpp index 1fc0999cac2e..8459d10339d0 100644 --- a/odbc/include/userver/storages/odbc/field.hpp +++ b/odbc/include/userver/storages/odbc/field.hpp @@ -3,14 +3,59 @@ /// @file userver/storages/odbc/field.hpp /// @brief @copybrief storages::odbc::Field +#include #include +#include +#include +#include +#include +#include +#include +#include #include +#include USERVER_NAMESPACE_BEGIN namespace storages::odbc { +/// @cond +namespace impl { + +template +struct IsOptional : std::false_type {}; + +template +struct IsOptional> : std::true_type { + using ValueType = T; +}; + +template +inline constexpr bool kIsOptional = IsOptional>::value; + +template +inline constexpr bool kIsFieldScalar = + std::same_as, bool> || + (std::integral> && !std::same_as, bool> && + sizeof(std::remove_cv_t) <= sizeof(std::uint64_t)) || + std::same_as, float> || std::same_as, double> || + std::same_as, std::string> || std::same_as, Bytes> || + std::same_as, Date> || std::same_as, Time> || + std::same_as, Timestamp> || kIsDecimal>; + +template +struct IsFieldAsType : std::bool_constant> {}; + +template +struct IsFieldAsType> : std::bool_constant> {}; + +template +inline constexpr bool kIsFieldAsType = IsFieldAsType>::value; + +} // namespace impl +/// @endcond + /// @brief Single cell in an ODBC result set row class Field { public: @@ -27,6 +72,11 @@ class Field { double GetDouble() const; bool GetBool() const; + /// Converts the field with strict SQL category, NULL and range checks. + /// Use `As>()` to accept SQL NULL. + template + T As() const; + protected: friend class Row; @@ -39,11 +89,74 @@ class Field { {} private: + std::int64_t GetSignedIntegerForAs() const; + std::uint64_t GetUnsignedIntegerForAs() const; + double GetFloatingPointForAs() const; + std::string GetStringForAs() const; + bool GetBoolForAs() const; + Bytes GetBytesForAs() const; + Date GetDateForAs() const; + Time GetTimeForAs() const; + Timestamp GetTimestampForAs() const; + std::string GetDecimalForAs(std::size_t precision, std::size_t scale) const; + detail::ResultWrapperPtr res_; size_type row_index_{0}; size_type field_index_{0}; }; +template +T Field::As() const { + using Value = std::remove_cv_t; + static_assert(impl::kIsFieldAsType, "Unsupported ODBC Field::As() type"); + + if constexpr (impl::kIsOptional) { + using Inner = typename impl::IsOptional::ValueType; + if (IsNull()) { + return std::nullopt; + } + return Value{As()}; + } else if constexpr (std::same_as) { + return GetBoolForAs(); + } else if constexpr (std::signed_integral) { + const auto value = GetSignedIntegerForAs(); + if (value < static_cast(std::numeric_limits::lowest()) || + value > static_cast(std::numeric_limits::max())) + { + throw ResultSetError("ODBC integer field does not fit into the requested signed type"); + } + return static_cast(value); + } else if constexpr (std::unsigned_integral) { + const auto value = GetUnsignedIntegerForAs(); + if (value > static_cast(std::numeric_limits::max())) { + throw ResultSetError("ODBC integer field does not fit into the requested unsigned type"); + } + return static_cast(value); + } else if constexpr (std::same_as) { + const auto value = GetFloatingPointForAs(); + if (value < static_cast(std::numeric_limits::lowest()) || + value > static_cast(std::numeric_limits::max())) + { + throw ResultSetError("ODBC floating-point field does not fit into float"); + } + return static_cast(value); + } else if constexpr (std::same_as) { + return GetFloatingPointForAs(); + } else if constexpr (std::same_as) { + return GetStringForAs(); + } else if constexpr (std::same_as) { + return GetBytesForAs(); + } else if constexpr (std::same_as) { + return GetDateForAs(); + } else if constexpr (std::same_as) { + return GetTimeForAs(); + } else if constexpr (std::same_as) { + return GetTimestampForAs(); + } else if constexpr (impl::kIsDecimal) { + return Value{GetDecimalForAs(Value::kPrecision, Value::kScale)}; + } +} + } // namespace storages::odbc USERVER_NAMESPACE_END diff --git a/odbc/include/userver/storages/odbc/impl/parameter.hpp b/odbc/include/userver/storages/odbc/impl/parameter.hpp index 019b620269c4..aeb586280e79 100644 --- a/odbc/include/userver/storages/odbc/impl/parameter.hpp +++ b/odbc/include/userver/storages/odbc/impl/parameter.hpp @@ -13,6 +13,7 @@ #include #include +#include #include USERVER_NAMESPACE_BEGIN @@ -25,14 +26,35 @@ enum class ParameterType { kUnsignedInteger, kFloatingPoint, kString, + kBytes, + kDate, + kTime, + kTimestamp, + kDecimal, kUnknown, }; +struct DecimalParameter final { + std::string representation; + std::uint8_t precision; + std::uint8_t scale; +}; + /// A type-erased, owning query parameter. Owning the value is important because /// an ODBC driver is allowed to read bound buffers until SQLExecute returns. class Parameter final { public: - using Value = std::variant; + using Value = std::variant< + bool, + std::int64_t, + std::uint64_t, + double, + std::string, + Bytes, + Date, + Time, + Timestamp, + DecimalParameter>; Parameter(std::nullptr_t) : type_{ParameterType::kUnknown}, @@ -87,6 +109,36 @@ class Parameter final { : Parameter{std::string{value}} {} + Parameter(Bytes value) + : type_{ParameterType::kBytes}, + value_{std::move(value)} + {} + + Parameter(Date value) + : type_{ParameterType::kDate}, + value_{value} + {} + + Parameter(Time value) + : type_{ParameterType::kTime}, + value_{value} + {} + + Parameter(Timestamp value) + : type_{ParameterType::kTimestamp}, + value_{value} + {} + + template + Parameter(const Decimal& value) + : type_{ParameterType::kDecimal}, + value_{DecimalParameter{ + std::string{value.GetRepresentation()}, + static_cast(Precision), + static_cast(Scale), + }} + {} + template Parameter(const std::optional& value) : Parameter{value ? Parameter{*value} : NullOf()} diff --git a/odbc/include/userver/storages/odbc/parameter_store.hpp b/odbc/include/userver/storages/odbc/parameter_store.hpp index 36ab46245326..6907bd7a0b1a 100644 --- a/odbc/include/userver/storages/odbc/parameter_store.hpp +++ b/odbc/include/userver/storages/odbc/parameter_store.hpp @@ -32,7 +32,8 @@ struct IsParameterStoreValue { public: static constexpr bool value = std::integral || std::floating_point || std::is_enum_v || - std::same_as || std::same_as || + std::same_as || std::same_as || std::same_as || + std::same_as || std::same_as || std::same_as || kIsDecimal || std::same_as || std::same_as || (std::is_pointer_v && (std::same_as || std::same_as)) || (std::is_array_v && std::same_as, char>); diff --git a/odbc/include/userver/storages/odbc/result_set.hpp b/odbc/include/userver/storages/odbc/result_set.hpp index 01644a14cbf6..5e352d754a53 100644 --- a/odbc/include/userver/storages/odbc/result_set.hpp +++ b/odbc/include/userver/storages/odbc/result_set.hpp @@ -3,10 +3,20 @@ /// @file userver/storages/odbc/result_set.hpp /// @brief @copybrief storages::odbc::ResultSet +#include +#include #include #include +#include +#include #include +#include +#include +#include +#include + +#include #include #include @@ -14,6 +24,49 @@ USERVER_NAMESPACE_BEGIN namespace storages::odbc { +/// @cond +namespace impl { + +struct OdbcResultMappingTag; + +template +constexpr bool AreResultMembersMappable(std::index_sequence) { + return sizeof...(Index) != 0 && + ((!std::is_reference_v> && + kIsFieldAsType>>) && + ...); +} + +template +constexpr bool DetectResultAggregate() { + using Value = std::remove_cv_t; + if constexpr (std::is_class_v && std::is_aggregate_v && std::is_standard_layout_v && + !std::is_union_v && boost::pfr::is_implicitly_reflectable_v && + !kIsFieldAsType) + { + return AreResultMembersMappable(std::make_index_sequence>{}); + } else { + return false; + } +} + +template +inline constexpr bool kIsResultAggregate = DetectResultAggregate(); + +template +inline constexpr bool kIsResultValue = kIsFieldAsType> || kIsResultAggregate; + +template +concept ResultContainer = + std::default_initializable && !std::same_as, std::string> && + requires { typename Container::value_type; } && kIsResultValue && + requires(Container& container, typename Container::value_type value) { + container.insert(container.end(), std::move(value)); + }; + +} // namespace impl +/// @endcond + /// @brief Result set for ODBC query execution class ResultSet final { public: @@ -49,10 +102,100 @@ class ResultSet final { reference operator[](size_type index) const&; + /// Materializes every row into the container's value type. Scalar values + /// require exactly one result column; aggregate values are initialized in + /// declaration order and require an exact column count. + template + requires impl::ResultContainer + Container AsContainer() const; + + /// Materializes the only result row, requiring exactly one row. + template + requires impl::kIsResultValue + T AsSingleRow() const; + + /// Returns no value for zero rows, materializes one row, and rejects more + /// than one row. For optional-valued T the outer optional represents row + /// presence and the inner optional represents SQL NULL. + template + requires impl::kIsResultValue + std::optional AsOptionalSingleRow() const; + private: + template + T MapAggregate(size_type row_index, std::index_sequence) const; + + template + T MapRow(size_type row_index) const; + std::shared_ptr pimpl_; }; +template +T ResultSet::MapAggregate(size_type row_index, std::index_sequence) const { + return T{ + operator[](row_index)[Index] + .template As(std::declval()))>>()... + }; +} + +template +T ResultSet::MapRow(size_type row_index) const { + using Value = std::remove_cv_t; + static_assert(impl::kIsResultValue, "Unsupported ODBC typed result value"); + + if constexpr (impl::kIsFieldAsType) { + if (FieldCount() != 1) { + throw ResultSetError("ODBC scalar result mapping requires exactly one column"); + } + return operator[](row_index)[0].template As(); + } else { + constexpr auto kFieldCount = boost::pfr::tuple_size_v; + if (FieldCount() != kFieldCount) { + throw ResultSetError("ODBC aggregate result mapping requires exactly one column per aggregate member"); + } + return MapAggregate(row_index, std::make_index_sequence{}); + } +} + +template +requires impl::ResultContainer +Container ResultSet::AsContainer() const { + using Value = typename Container::value_type; + static_assert(impl::kIsResultValue, "Unsupported ODBC typed result container value"); + + Container result; + if constexpr (requires { result.reserve(Size()); }) { + result.reserve(Size()); + } + auto output = std::inserter(result, result.end()); + for (size_type index = 0; index < Size(); ++index) { + *output++ = MapRow(index); + } + return result; +} + +template +requires impl::kIsResultValue +T ResultSet::AsSingleRow() const { + if (Size() != 1) { + throw ResultSetError("ODBC single-row result mapping requires exactly one row"); + } + return MapRow(0); +} + +template +requires impl::kIsResultValue +std::optional ResultSet::AsOptionalSingleRow() const { + if (Size() > 1) { + throw ResultSetError("ODBC optional single-row result mapping accepts at most one row"); + } + if (IsEmpty()) { + return std::nullopt; + } + return std::optional{MapRow(0)}; +} + } // namespace storages::odbc USERVER_NAMESPACE_END diff --git a/odbc/include/userver/storages/odbc/types.hpp b/odbc/include/userver/storages/odbc/types.hpp new file mode 100644 index 000000000000..3831a06768d0 --- /dev/null +++ b/odbc/include/userver/storages/odbc/types.hpp @@ -0,0 +1,251 @@ +#pragma once + +/// @file userver/storages/odbc/types.hpp +/// @brief Portable value types for standard ODBC SQL types. + +#include +#include +#include +#include +#include +#include +#include +#include + +USERVER_NAMESPACE_BEGIN + +namespace storages::odbc { + +/// Owning byte sequence for SQL BINARY, VARBINARY and LONGVARBINARY. +class Bytes final { +public: + using ValueType = std::uint8_t; + using Container = std::vector; + + Bytes() = default; + explicit Bytes(Container bytes); + Bytes(std::initializer_list bytes); + + const Container& GetBytes() const noexcept; + std::size_t Size() const noexcept; + bool IsEmpty() const noexcept; + + bool operator==(const Bytes&) const noexcept = default; + +private: + Container bytes_; +}; + +/// Timezone-independent Gregorian calendar date in the portable 1..9999 range. +class Date final { +public: + Date() noexcept; + Date(std::uint32_t year, std::uint32_t month, std::uint32_t day); + + std::uint32_t GetYear() const noexcept; + std::uint32_t GetMonth() const noexcept; + std::uint32_t GetDay() const noexcept; + std::string ToString() const; + + bool operator==(const Date&) const noexcept = default; + +private: + std::uint16_t year_{1970}; + std::uint8_t month_{1}; + std::uint8_t day_{1}; +}; + +/// Timezone-independent time of day with the portable `SQL_TIME_STRUCT` +/// resolution of one second. +class Time final { +public: + Time() noexcept = default; + Time(std::uint32_t hour, std::uint32_t minute, std::uint32_t second); + + std::uint32_t GetHour() const noexcept; + std::uint32_t GetMinute() const noexcept; + std::uint32_t GetSecond() const noexcept; + std::string ToString() const; + + bool operator==(const Time&) const noexcept = default; + +private: + std::uint8_t hour_{0}; + std::uint8_t minute_{0}; + std::uint8_t second_{0}; +}; + +/// Timezone-independent timestamp with nanosecond fraction storage. +/// +/// No implicit conversion to or from `std::chrono::system_clock::time_point` +/// is provided because an ODBC TIMESTAMP has no timezone. +class Timestamp final { +public: + Timestamp() noexcept = default; + Timestamp(Date date, Time time, std::uint32_t fraction_nanoseconds = 0); + Timestamp( + std::uint32_t year, + std::uint32_t month, + std::uint32_t day, + std::uint32_t hour, + std::uint32_t minute, + std::uint32_t second, + std::uint32_t fraction_nanoseconds = 0 + ); + + const Date& GetDate() const noexcept; + const Time& GetTime() const noexcept; + std::uint32_t GetFractionNanoseconds() const noexcept; + std::string ToString() const; + + bool operator==(const Timestamp&) const noexcept = default; + +private: + Date date_; + Time time_; + std::uint32_t fraction_nanoseconds_{0}; +}; + +/// Exact fixed-point SQL DECIMAL/NUMERIC value. +/// +/// Accepted syntax is `[-+]digits` for Scale=0 and +/// `[-+]digits.Scale-digits` otherwise. Exponents, whitespace, NaN and +/// infinities are rejected. Values are canonicalized by removing a leading +/// plus and redundant integer zeroes; negative zero is normalized to positive +/// zero. Exactly Scale fractional digits, including trailing zeroes, are +/// retained. ODBC SQL_NUMERIC_STRUCT limits portable precision to 38 digits. +template +class Decimal final { + static_assert(Precision >= 1 && Precision <= 38, "ODBC Decimal precision must be in the range 1..38"); + static_assert(Scale <= Precision, "ODBC Decimal scale must not exceed precision"); + +public: + static constexpr std::size_t kPrecision = Precision; + static constexpr std::size_t kScale = Scale; + + Decimal() + : representation_{MakeZero()} + {} + + explicit Decimal(std::string_view representation) + : representation_{Validate(representation)} + {} + + std::string_view GetRepresentation() const noexcept; + static constexpr std::size_t GetPrecision() noexcept { return Precision; } + static constexpr std::size_t GetScale() noexcept { return Scale; } + + bool operator==(const Decimal&) const noexcept = default; + +private: + static std::string MakeZero(); + static std::string Validate(std::string_view representation); + + std::string representation_; +}; + +/// @cond +namespace impl { + +template +struct IsDecimal : std::false_type {}; + +template +struct IsDecimal> : std::true_type {}; + +template +inline constexpr bool kIsDecimal = IsDecimal>::value; + +} // namespace impl +/// @endcond + +template +std::string_view Decimal::GetRepresentation() const noexcept { + return representation_; +} + +template +std::string Decimal::MakeZero() { + if constexpr (Scale == 0) { + return "0"; + } else { + return std::string{"0."} + std::string(Scale, '0'); + } +} + +template +std::string Decimal::Validate(std::string_view representation) { + if (representation.empty()) { + throw std::invalid_argument("ODBC Decimal representation must not be empty"); + } + + std::size_t index = representation.front() == '-' || representation.front() == '+' ? 1 : 0; + const auto integer_begin = index; + while (index < representation.size() && representation[index] >= '0' && representation[index] <= '9') { + ++index; + } + if (index == integer_begin) { + throw std::invalid_argument("ODBC Decimal requires at least one integer digit"); + } + const auto integer_end = index; + + if constexpr (Scale == 0) { + if (index != representation.size()) { + throw std::invalid_argument("ODBC Decimal with scale 0 must not contain a fractional part"); + } + } else { + if (index == representation.size() || representation[index] != '.') { + throw std::invalid_argument("ODBC Decimal representation does not contain its declared scale"); + } + ++index; + const auto fractional_begin = index; + while (index < representation.size() && representation[index] >= '0' && representation[index] <= '9') { + ++index; + } + if (index != representation.size() || index - fractional_begin != Scale) { + throw std::invalid_argument("ODBC Decimal fractional digits do not match its declared scale"); + } + } + + auto first_significant = integer_begin; + while (first_significant < integer_end && representation[first_significant] == '0') { + ++first_significant; + } + const auto significant_integer_digits = first_significant == integer_end ? 0 : integer_end - first_significant; + if (significant_integer_digits > Precision - Scale) { + throw std::out_of_range("ODBC Decimal magnitude exceeds its declared precision and scale"); + } + + const bool fractional_is_zero = [&] { + if constexpr (Scale == 0) { + return true; + } else { + for (std::size_t position = integer_end + 1; position < representation.size(); ++position) { + if (representation[position] != '0') { + return false; + } + } + return true; + } + }(); + const bool is_zero = significant_integer_digits == 0 && fractional_is_zero; + + std::string result; + if (!is_zero && representation.front() == '-') { + result.push_back('-'); + } + if (first_significant == integer_end) { + result.push_back('0'); + } else { + result.append(representation.substr(first_significant, significant_integer_digits)); + } + if constexpr (Scale != 0) { + result.push_back('.'); + result.append(representation.substr(integer_end + 1, Scale)); + } + return result; +} + +} // namespace storages::odbc + +USERVER_NAMESPACE_END diff --git a/odbc/src/storages/odbc/detail/connection.cpp b/odbc/src/storages/odbc/detail/connection.cpp index 6682dce377b5..4f6fa45748c5 100644 --- a/odbc/src/storages/odbc/detail/connection.cpp +++ b/odbc/src/storages/odbc/detail/connection.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -29,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -105,6 +107,16 @@ Exception MakeDriverError(std::string message, SQLRETURN result, SQLHANDLE handl return Exception{std::move(message), std::move(diagnostics), result == SQL_INVALID_HANDLE}; } +template +Exception MakeDiagnosticError(std::string message, std::vector diagnostics) { + const auto formatted = detail::FormatSQLDiagnostics(diagnostics); + if (!formatted.empty()) { + message += ": "; + message += formatted; + } + return Exception{std::move(message), std::move(diagnostics)}; +} + ConnectionError MakeConnectionError(std::string message, SQLRETURN result, std::vector diagnostics) { const auto formatted = detail::FormatSQLDiagnostics(diagnostics); if (!formatted.empty()) { @@ -125,6 +137,21 @@ void LogConnectionAttributeWarnings(std::string_view operation, const std::vecto << operation << " completed with warning: " << (formatted.empty() ? "no diagnostic records" : formatted); } +void LogOdbcWarnings(std::string_view operation, const std::vector& diagnostics) { + constexpr std::size_t kMaxWarningLength = 1024; + auto formatted = detail::FormatSQLDiagnostics(diagnostics); + if (formatted.size() > kMaxWarningLength) { + formatted.resize(kMaxWarningLength); + formatted += "..."; + } + LOG_WARNING() + << operation << " completed with warning: " << (formatted.empty() ? "no diagnostic records" : formatted); +} + +void LogOdbcWarnings(std::string_view operation, SQLHANDLE handle, SQLSMALLINT handle_type) { + LogOdbcWarnings(operation, detail::GetSQLDiagnostics(handle, handle_type)); +} + SQLUINTEGER ReadConnectionAttribute( SQLHDBC connection, SQLINTEGER attribute, @@ -309,10 +336,20 @@ void CheckStatementResult(SQLRETURN result, SQLHSTMT statement, std::string_view } } -detail::ResultWrapper::Column DescribeColumn(SQLHSTMT statement, SQLUSMALLINT column) { +bool IsTruncationWarning(const std::vector& diagnostics); +bool HasNonTruncationWarning(const std::vector& diagnostics); +bool HasDataLossWarning(const std::vector& diagnostics); +std::vector GetWarnings(SQLRETURN result, SQLHANDLE handle, SQLSMALLINT handle_type); + +void HandleStatementWarnings(SQLRETURN result, SQLHSTMT statement, std::string_view operation, bool reject_data_loss); + +detail::ResultWrapper::Column DescribeColumn(SQLHSTMT statement, SQLUSMALLINT column, engine::Deadline deadline) { std::array buffer{}; SQLSMALLINT name_length = 0; SQLSMALLINT type = SQL_UNKNOWN_TYPE; + SQLULEN size = 0; + SQLSMALLINT decimal_digits = 0; + detail::CheckDeadlineNotExpired(deadline); auto result = SQLDescribeCol( statement, column, @@ -320,14 +357,20 @@ detail::ResultWrapper::Column DescribeColumn(SQLHSTMT statement, SQLUSMALLINT co static_cast(buffer.size()), &name_length, &type, - nullptr, - nullptr, + &size, + &decimal_digits, nullptr ); + auto warnings = GetWarnings(result, statement, SQL_HANDLE_STMT); + detail::CheckDeadlineNotExpired(deadline); CheckStatementResult(result, statement, "describe result column"); if (name_length >= static_cast(buffer.size())) { + if (result == SQL_SUCCESS_WITH_INFO && HasNonTruncationWarning(warnings)) { + LogOdbcWarnings("Describing ODBC result column", warnings); + } std::vector long_buffer(static_cast(name_length) + 1); + detail::CheckDeadlineNotExpired(deadline); result = SQLDescribeCol( statement, column, @@ -335,33 +378,96 @@ detail::ResultWrapper::Column DescribeColumn(SQLHSTMT statement, SQLUSMALLINT co static_cast(long_buffer.size()), &name_length, &type, - nullptr, - nullptr, + &size, + &decimal_digits, nullptr ); + warnings = GetWarnings(result, statement, SQL_HANDLE_STMT); + detail::CheckDeadlineNotExpired(deadline); CheckStatementResult(result, statement, "describe result column"); + if (result == SQL_SUCCESS_WITH_INFO) { + if (HasDataLossWarning(warnings)) { + throw MakeDiagnosticError( + "ODBC driver truncated a result column description after an exact-size retry", + std::move(warnings) + ); + } + LogOdbcWarnings("Describing ODBC result column after an exact-size retry", warnings); + } return { std::string{reinterpret_cast(long_buffer.data()), static_cast(name_length)}, type, + size, + decimal_digits, }; } + if (result == SQL_SUCCESS_WITH_INFO) { + if (HasDataLossWarning(warnings)) { + throw MakeDiagnosticError( + "ODBC driver truncated a result column description without reporting a larger name", + std::move(warnings) + ); + } + LogOdbcWarnings("Describing ODBC result column", warnings); + } + return { std::string{reinterpret_cast(buffer.data()), static_cast(name_length)}, type, + size, + decimal_digits, }; } -detail::ResultWrapper::Cell ReadCell(SQLHSTMT statement, SQLUSMALLINT column, engine::Deadline deadline) { +bool IsTruncationWarning(const std::vector& diagnostics) { + return std::any_of(diagnostics.begin(), diagnostics.end(), [](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state == "01004"; + }); +} + +bool HasNonTruncationWarning(const std::vector& diagnostics) { + return std::any_of(diagnostics.begin(), diagnostics.end(), [](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state != "01004"; + }); +} + +bool HasDataLossWarning(const std::vector& diagnostics) { + return std::any_of(diagnostics.begin(), diagnostics.end(), [](const DiagnosticRecord& diagnostic) { + return diagnostic.sql_state == "01004" || diagnostic.sql_state == "01S07" || diagnostic.sql_state == "22003"; + }); +} + +std::vector GetWarnings(SQLRETURN result, SQLHANDLE handle, SQLSMALLINT handle_type) { + return result == SQL_SUCCESS_WITH_INFO + ? detail::GetSQLDiagnostics(handle, handle_type) + : std::vector{}; +} + +void HandleStatementWarnings(SQLRETURN result, SQLHSTMT statement, std::string_view operation, bool reject_data_loss) { + if (result != SQL_SUCCESS_WITH_INFO) { + return; + } + auto diagnostics = detail::GetSQLDiagnostics(statement, SQL_HANDLE_STMT); + if (reject_data_loss && HasDataLossWarning(diagnostics)) { + throw MakeDiagnosticError< + StatementError>(fmt::format("ODBC driver lost information while {}", operation), std::move(diagnostics)); + } + LogOdbcWarnings(fmt::format("ODBC statement {}", operation), diagnostics); +} + +detail::ResultWrapper::Cell ReadTextCell(SQLHSTMT statement, SQLUSMALLINT column, engine::Deadline deadline) { constexpr std::size_t kChunkSize = 4096; std::array buffer{}; std::string value; + std::optional previous_remaining; while (true) { SQLLEN indicator = 0; detail::CheckDeadlineNotExpired(deadline); const auto result = SQLGetData(statement, column, SQL_C_CHAR, buffer.data(), static_cast(buffer.size()), &indicator); + const auto warnings = GetWarnings(result, statement, SQL_HANDLE_STMT); detail::CheckDeadlineNotExpired(deadline); if (result == SQL_NO_DATA) { @@ -371,37 +477,329 @@ detail::ResultWrapper::Cell ReadCell(SQLHSTMT statement, SQLUSMALLINT column, en if (indicator == SQL_NULL_DATA) { return {std::nullopt}; } + const bool truncated = IsTruncationWarning(warnings); + const auto + chunk = detail::AccountResultChunk(result, indicator, truncated, buffer.size() - 1, previous_remaining); + value.append(reinterpret_cast(buffer.data()), chunk.size); + previous_remaining = chunk.known_remaining; + if (chunk.has_more) { + if (HasNonTruncationWarning(warnings)) { + LogOdbcWarnings("Reading ODBC character result", warnings); + } + continue; + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogOdbcWarnings("Reading ODBC character result", warnings); + } + if (result == SQL_SUCCESS || result == SQL_SUCCESS_WITH_INFO) { + break; + } + } + + return {detail::ResultWrapper::Cell::Value{std::move(value)}}; +} - const auto terminator = std::find(buffer.begin(), buffer.end(), static_cast('\0')); - const auto chunk_size = static_cast(terminator - buffer.begin()); - value.append(reinterpret_cast(buffer.data()), chunk_size); +detail::ResultWrapper::Cell ReadBytesCell(SQLHSTMT statement, SQLUSMALLINT column, engine::Deadline deadline) { + constexpr std::size_t kChunkSize = 4096; + std::array buffer{}; + Bytes::Container value; + std::optional previous_remaining; - if (result == SQL_SUCCESS) { + while (true) { + SQLLEN indicator = 0; + detail::CheckDeadlineNotExpired(deadline); + const auto result = + SQLGetData(statement, column, SQL_C_BINARY, buffer.data(), static_cast(buffer.size()), &indicator); + const auto warnings = GetWarnings(result, statement, SQL_HANDLE_STMT); + detail::CheckDeadlineNotExpired(deadline); + + if (result == SQL_NO_DATA) { break; } - if (chunk_size == 0) { - throw ResultSetError("ODBC driver returned SQL_SUCCESS_WITH_INFO without result data progress"); + CheckStatementResult(result, statement, "read binary result column"); + if (indicator == SQL_NULL_DATA) { + return {std::nullopt}; + } + const bool truncated = IsTruncationWarning(warnings); + const auto chunk = detail::AccountResultChunk(result, indicator, truncated, buffer.size(), previous_remaining); + value.insert(value.end(), buffer.begin(), buffer.begin() + static_cast(chunk.size)); + previous_remaining = chunk.known_remaining; + if (chunk.has_more) { + if (HasNonTruncationWarning(warnings)) { + LogOdbcWarnings("Reading ODBC binary result", warnings); + } + continue; + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogOdbcWarnings("Reading ODBC binary result", warnings); + } + if (result == SQL_SUCCESS || result == SQL_SUCCESS_WITH_INFO) { + break; } } - return {std::move(value)}; + return {detail::ResultWrapper::Cell::Value{Bytes{std::move(value)}}}; +} + +template +detail::ResultWrapper::Cell ReadFixedCell( + SQLHSTMT statement, + SQLUSMALLINT column, + SQLSMALLINT c_type, + std::string_view type_name, + engine::Deadline deadline, + ValueFactory&& value_factory +) { + Struct value{}; + SQLLEN indicator = 0; + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLGetData(statement, column, c_type, &value, static_cast(sizeof(value)), &indicator); + auto warnings = GetWarnings(result, statement, SQL_HANDLE_STMT); + detail::CheckDeadlineNotExpired(deadline); + CheckStatementResult(result, statement, fmt::format("read {} result column", type_name)); + if (indicator == SQL_NULL_DATA) { + return {std::nullopt}; + } + detail::ValidateFixedResultSize(indicator, sizeof(value)); + if (result == SQL_SUCCESS_WITH_INFO) { + if (HasDataLossWarning(warnings)) { + throw MakeDiagnosticError( + fmt::format("ODBC driver lost information while converting a fixed-size {} result", type_name), + std::move(warnings) + ); + } + LogOdbcWarnings(fmt::format("Reading ODBC {} result", type_name), warnings); + } + try { + return {detail::ResultWrapper::Cell::Value{std::forward(value_factory)(value)}}; + } catch (const std::invalid_argument& ex) { + throw ResultSetError(fmt::format("ODBC driver returned an invalid {} value: {}", type_name, ex.what())); + } +} + +std::string DecodeNumericMagnitude(const SQL_NUMERIC_STRUCT& value) { + std::vector decimal_digits{0}; + for (std::size_t byte_index = sizeof(value.val); byte_index > 0; --byte_index) { + unsigned carry = value.val[byte_index - 1]; + for (auto& digit : decimal_digits) { + const auto next = static_cast(digit) * 256U + carry; + digit = static_cast(next % 10U); + carry = next / 10U; + } + while (carry != 0) { + decimal_digits.push_back(static_cast(carry % 10U)); + carry /= 10U; + } + } + while (decimal_digits.size() > 1 && decimal_digits.back() == 0) { + decimal_digits.pop_back(); + } + std::string result; + result.reserve(decimal_digits.size()); + for (auto iterator = decimal_digits.rbegin(); iterator != decimal_digits.rend(); ++iterator) { + result.push_back(static_cast('0' + *iterator)); + } + return result; +} + +detail::ResultWrapper::Cell ReadDecimalCell( + SQLHSTMT statement, + SQLUSMALLINT column, + const detail::ResultWrapper::Column& metadata, + engine::Deadline deadline +) { + if (metadata.size == 0 || metadata.size > 38 || metadata.decimal_digits < 0 || + static_cast(metadata.decimal_digits) > metadata.size) + { + throw ResultSetError(fmt::format( + "ODBC Decimal column {} has invalid or unknown precision/scale metadata: precision {}, scale {}", + column, + metadata.size, + metadata.decimal_digits + )); + } + + SQLHDESC descriptor = SQL_NULL_HDESC; + detail::CheckDeadlineNotExpired(deadline); + const auto descriptor_result = SQLGetStmtAttr( + statement, + SQL_ATTR_APP_ROW_DESC, + &descriptor, + static_cast(sizeof(descriptor)), + nullptr + ); + detail::CheckDeadlineNotExpired(deadline); + if (!SQL_SUCCEEDED(descriptor_result)) { + throw MakeDriverError( + "Failed to obtain the descriptor for an ODBC Decimal result", + descriptor_result, + statement, + SQL_HANDLE_STMT + ); + } + if (descriptor_result == SQL_SUCCESS_WITH_INFO) { + LogOdbcWarnings("Obtaining the descriptor for an ODBC Decimal result", statement, SQL_HANDLE_STMT); + } + + const auto set_field = [&](SQLSMALLINT field, SQLLEN value, std::string_view name) { + detail::CheckDeadlineNotExpired(deadline); + const auto result = SQLSetDescField( + descriptor, + static_cast(column), + field, + reinterpret_cast(static_cast(value)), + SQL_IS_INTEGER + ); + detail::CheckDeadlineNotExpired(deadline); + if (!SQL_SUCCEEDED(result)) { + throw MakeDriverError( + fmt::format("Failed to set {} for an ODBC Decimal result", name), + result, + descriptor, + SQL_HANDLE_DESC + ); + } + if (result == SQL_SUCCESS_WITH_INFO) { + LogOdbcWarnings(fmt::format("Setting {} for an ODBC Decimal result", name), descriptor, SQL_HANDLE_DESC); + } + }; + set_field(SQL_DESC_CONCISE_TYPE, SQL_C_NUMERIC, "C type"); + set_field(SQL_DESC_PRECISION, static_cast(metadata.size), "precision"); + set_field(SQL_DESC_SCALE, metadata.decimal_digits, "scale"); + + SQL_NUMERIC_STRUCT value{}; + SQLLEN indicator = 0; + detail::CheckDeadlineNotExpired(deadline); + const auto + result = SQLGetData(statement, column, SQL_ARD_TYPE, &value, static_cast(sizeof(value)), &indicator); + auto warnings = GetWarnings(result, statement, SQL_HANDLE_STMT); + detail::CheckDeadlineNotExpired(deadline); + CheckStatementResult(result, statement, "read Decimal result column"); + if (indicator == SQL_NULL_DATA) { + return {std::nullopt}; + } + detail::ValidateFixedResultSize(indicator, sizeof(value)); + if (result == SQL_SUCCESS_WITH_INFO) { + if (HasDataLossWarning(warnings)) { + throw MakeDiagnosticError< + ResultSetError>("ODBC driver lost information while converting a Decimal result", std::move(warnings)); + } + LogOdbcWarnings("Reading ODBC Decimal result", warnings); + } + if (value.sign != 0 && value.sign != 1) { + throw ResultSetError("ODBC driver returned an invalid SQL_NUMERIC_STRUCT sign"); + } + if (value.precision == 0 || value.precision > metadata.size || value.scale != metadata.decimal_digits) { + throw ResultSetError(fmt::format( + "ODBC driver returned invalid Decimal value precision/scale {},{} for column precision/scale {},{}", + value.precision, + value.scale, + metadata.size, + metadata.decimal_digits + )); + } + + auto representation = DecodeNumericMagnitude(value); + const bool is_zero = representation == "0"; + const auto scale = static_cast(metadata.decimal_digits); + detail::ValidateNumericMagnitude(representation.size(), metadata.size, scale); + if (scale != 0) { + if (representation.size() <= scale) { + representation.insert(0, scale + 1 - representation.size(), '0'); + } + representation.insert(representation.size() - scale, 1, '.'); + } + if (value.sign == 0 && !is_zero) { + representation.insert(representation.begin(), '-'); + } + return {detail::ResultWrapper::Cell::Value{detail::ResultWrapper::DecimalValue{ + std::move(representation), + static_cast(metadata.size), + static_cast(metadata.decimal_digits), + }}}; +} + +detail::ResultWrapper::Cell ReadCell( + SQLHSTMT statement, + SQLUSMALLINT column, + const detail::ResultWrapper::Column& metadata, + engine::Deadline deadline +) { + if (metadata.type == SQL_TYPE_DATE || metadata.type == SQL_DATE) { + return ReadFixedCell< + SQL_DATE_STRUCT>(statement, column, SQL_C_TYPE_DATE, "Date", deadline, [](const SQL_DATE_STRUCT& value) { + return Date{ + static_cast(value.year), + static_cast(value.month), + static_cast(value.day), + }; + }); + } + if (metadata.type == SQL_TYPE_TIME || metadata.type == SQL_TIME) { + return ReadFixedCell< + SQL_TIME_STRUCT>(statement, column, SQL_C_TYPE_TIME, "Time", deadline, [](const SQL_TIME_STRUCT& value) { + return Time{ + static_cast(value.hour), + static_cast(value.minute), + static_cast(value.second), + }; + }); + } + if (metadata.type == SQL_TYPE_TIMESTAMP || metadata.type == SQL_TIMESTAMP) { + return ReadFixedCell( + statement, + column, + SQL_C_TYPE_TIMESTAMP, + "Timestamp", + deadline, + [](const SQL_TIMESTAMP_STRUCT& value) { + return Timestamp{ + static_cast(value.year), + value.month, + value.day, + value.hour, + value.minute, + value.second, + value.fraction, + }; + } + ); + } + switch (metadata.type) { + case SQL_BINARY: + case SQL_VARBINARY: + case SQL_LONGVARBINARY: + return ReadBytesCell(statement, column, deadline); + case SQL_DECIMAL: + case SQL_NUMERIC: + return ReadDecimalCell(statement, column, metadata, deadline); + default: + return ReadTextCell(statement, column, deadline); + } } std::shared_ptr MaterializeResult(SQLHSTMT statement, engine::Deadline deadline) { SQLSMALLINT column_count = 0; - CheckStatementResult(SQLNumResultCols(statement, &column_count), statement, "get result column count for"); + detail::CheckDeadlineNotExpired(deadline); + const auto column_count_result = SQLNumResultCols(statement, &column_count); + HandleStatementWarnings(column_count_result, statement, "determining result column count", false); + detail::CheckDeadlineNotExpired(deadline); + CheckStatementResult(column_count_result, statement, "get result column count for"); std::size_t rows_affected = 0; if (column_count == 0) { SQLLEN affected = 0; + detail::CheckDeadlineNotExpired(deadline); const auto row_count_result = SQLRowCount(statement, &affected); + HandleStatementWarnings(row_count_result, statement, "determining the affected row count", false); + detail::CheckDeadlineNotExpired(deadline); rows_affected = SQL_SUCCEEDED(row_count_result) && affected > 0 ? static_cast(affected) : 0; } std::vector columns; columns.reserve(static_cast(column_count)); for (SQLSMALLINT index = 0; index < column_count; ++index) { - columns.push_back(DescribeColumn(statement, static_cast(index + 1))); + columns.push_back(DescribeColumn(statement, static_cast(index + 1), deadline)); } std::vector rows; @@ -409,6 +807,7 @@ std::shared_ptr MaterializeResult(SQLHSTMT statement, eng while (true) { detail::CheckDeadlineNotExpired(deadline); const auto fetch_result = SQLFetch(statement); + HandleStatementWarnings(fetch_result, statement, "fetching a result row", false); detail::CheckDeadlineNotExpired(deadline); if (fetch_result == SQL_NO_DATA) { break; @@ -418,7 +817,12 @@ std::shared_ptr MaterializeResult(SQLHSTMT statement, eng detail::ResultWrapper::Row row; row.reserve(static_cast(column_count)); for (SQLSMALLINT index = 0; index < column_count; ++index) { - row.push_back(ReadCell(statement, static_cast(index + 1), deadline)); + row.push_back(ReadCell( + statement, + static_cast(index + 1), + columns[static_cast(index)], + deadline + )); } rows.push_back(std::move(row)); } @@ -431,12 +835,56 @@ struct ParameterBinding final { SQLSMALLINT c_type; SQLSMALLINT sql_type; SQLULEN column_size; + SQLSMALLINT decimal_digits; SQLPOINTER data; SQLLEN buffer_size; }; +struct BoundBytes final { + explicit BoundBytes(const Bytes& value) + : bytes{value.GetBytes().begin(), value.GetBytes().end()} + {} + + SQLPOINTER Data() noexcept { return bytes.empty() ? &empty_value : bytes.data(); } + + std::vector bytes; + SQLCHAR empty_value{0}; +}; + +SQL_NUMERIC_STRUCT MakeNumericStruct(const impl::DecimalParameter& parameter) { + SQL_NUMERIC_STRUCT result{}; + result.precision = parameter.precision; + result.scale = static_cast(parameter.scale); + result.sign = parameter.representation.front() == '-' ? 0 : 1; + + for (const char ch : parameter.representation) { + if (ch == '-' || ch == '+' || ch == '.') { + continue; + } + unsigned carry = static_cast(ch - '0'); + for (auto& byte : result.val) { + const auto value = static_cast(byte) * 10U + carry; + byte = static_cast(value & 0xffU); + carry = value >> 8U; + } + if (carry != 0) { + throw StatementError("ODBC Decimal magnitude exceeds SQL_NUMERIC_STRUCT capacity"); + } + } + return result; +} + struct BoundParameter final { - using Value = std::variant; + using Value = std::variant< + SQLCHAR, + SQLBIGINT, + SQLDOUBLE, + std::string, + BoundBytes, + SQL_DATE_STRUCT, + SQL_TIME_STRUCT, + SQL_TIMESTAMP_STRUCT, + SQL_NUMERIC_STRUCT>; explicit BoundParameter(const impl::Parameter& parameter) : type{parameter.GetType()}, @@ -463,6 +911,40 @@ struct BoundParameter final { case ParameterType::kString: case ParameterType::kUnknown: return parameter.Get(); + case ParameterType::kBytes: + return BoundBytes{parameter.Get()}; + case ParameterType::kDate: { + const auto& date = parameter.Get(); + return SQL_DATE_STRUCT{ + static_cast(date.GetYear()), + static_cast(date.GetMonth()), + static_cast(date.GetDay()), + }; + } + case ParameterType::kTime: { + const auto& time = parameter.Get