Skip to content

Commit 1ea5e73

Browse files
theorytheory
committed
Add support for the Decimal data type
The mappings already work for the HTTP engine, although its inserts were limited to `float8`, now fixed to use proper numeric formatting. Add the mappings for the binary engine. Use strings to convert between the Postgres and ClickHouse formats to as to avoid differences in their implementations: ClickHouse stores data as `Int32`, `Int64`, or `Int128`, while Postgres...well, I have no idea what the internals are, but using the `numeric_in` and `numeric_out` functions avoid all issues. Conversion from `Decimal` values returned from ClickHouse requires converting its internal `Int128` into a string. The clickhouse-cpp library doesn't provide this functionality, [yet], so implement it here. Unfortunately, this approach doesn't work with clickhouse-cpp compiled into a dynamic library, presumably because it doesn't also compile and install the `absl` dynamic library. So make the build static by default on all platforms for now. Add a test that covers both the HTTP and binary engines. [yet]: ClickHouse/clickhouse-cpp#451
1 parent 372652f commit 1ea5e73

File tree

5 files changed

+230
-8
lines changed

5 files changed

+230
-8
lines changed

‎Makefile‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -32,9 +32,9 @@ CH_CPP_FLAGS = -D CMAKE_BUILD_TYPE=Release -D WITH_OPENSSL=ON
3232

3333
# Build static on Darwin by default.
3434
ifndef ($(CH_BUILD))
35-
ifeq ($(OS),Darwin)
35+
# ifeq ($(OS),Darwin)
3636
CH_BUILD = static
37-
endif
37+
# endif
3838
endif
3939

4040
# Are we statically compiling clickhouse-cpp into the extension or no?

‎src/binary.cpp‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
#include <sstream>
12
#include <iostream>
23
#include <cassert>
34
#include <stdexcept>
@@ -250,6 +251,11 @@ static Oid get_corr_postgres_type(const TypeRef & type)
250251
return FLOAT4OID;
251252
case Type::Code::Float64:
252253
return FLOAT8OID;
254+
case Type::Code::Decimal128:
255+
case Type::Code::Decimal64:
256+
case Type::Code::Decimal32:
257+
case Type::Code::Decimal:
258+
return NUMERICOID;
253259
case Type::Code::FixedString:
254260
case Type::Code::Enum8:
255261
case Type::Code::Enum16:
@@ -461,6 +467,26 @@ static void column_append(clickhouse::ColumnRef col, Datum val, Oid valtype, boo
461467
}
462468
break;
463469
}
470+
case NUMERICOID: {
471+
// Convert numeric to string and let ColumnDecimal parse it.
472+
char *s = DatumGetCString(DirectFunctionCall1(numeric_out, val));
473+
switch (col->Type()->GetCode())
474+
{
475+
case Type::Code::Decimal128:
476+
case Type::Code::Decimal64:
477+
case Type::Code::Decimal32:
478+
case Type::Code::Decimal:
479+
col->As<ColumnDecimal>()->Append(std::string(s));
480+
break;
481+
default:
482+
throw std::runtime_error(
483+
"unexpected column "
484+
"type for NUMERIC: "
485+
+ col->Type()->GetName());
486+
}
487+
pfree(s);
488+
break;
489+
}
464490
case TEXTOID: {
465491
char * s = TextDatumGetCString(val);
466492

@@ -745,6 +771,55 @@ static Datum make_datum(clickhouse::ColumnRef col, size_t row, Oid * valtype, bo
745771
*valtype = FLOAT8OID;
746772
}
747773
break;
774+
case Type::Code::Decimal128:
775+
case Type::Code::Decimal64:
776+
case Type::Code::Decimal32:
777+
case Type::Code::Decimal:
778+
{
779+
auto decCol = col->As<ColumnDecimal>();
780+
auto val = decCol->At(row);
781+
782+
// Convert the Int128 to a string.
783+
std::stringstream ss;
784+
ss << val;
785+
std::string str = ss.str();
786+
787+
// Start a destination string.
788+
std::stringstream res;
789+
auto scale = decCol->GetScale();
790+
791+
// Output a dash for negative values
792+
if (val < 0)
793+
{
794+
res << '-';
795+
str.erase(0, 1);
796+
}
797+
798+
if (str.length() <= scale)
799+
{
800+
// Append the entire value prepended with zeros after the decimal.
801+
res << "0." << std::string(scale-1, '0') << str;
802+
}
803+
else
804+
{
805+
// There are digits before the decimal.
806+
auto decAt = str.length() - scale;
807+
res << str.substr(0, decAt);
808+
809+
// Append any digits after the decimal.
810+
if (decAt < str.length())
811+
{
812+
res << '.' << str.substr(decAt);
813+
}
814+
}
815+
816+
ret = DirectFunctionCall3(numeric_in,
817+
CStringGetDatum(res.str().c_str()),
818+
ObjectIdGetDatum(0),
819+
Int32GetDatum(-1));
820+
*valtype = NUMERICOID;
821+
}
822+
break;
748823
case Type::Code::FixedString: {
749824
auto s = std::string(col->As<ColumnFixedString>()->At(row));
750825
ret = CStringGetTextDatum(s.c_str());

‎src/pglink.c‎

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -377,12 +377,9 @@ extend_insert_query(ch_http_insert_state *state, TupleTableSlot *slot)
377377
break;
378378
case NUMERICOID:
379379
{
380-
Datum valueDatum;
381-
float8 f;
382-
383-
valueDatum = DirectFunctionCall1(numeric_float8, value);
384-
f = DatumGetFloat8(valueDatum);
385-
appendStringInfo(&state->sql, "%f", f);
380+
char *extval = DatumGetCString(DirectFunctionCall1(numeric_out, value));
381+
appendStringInfoString(&state->sql, extval);
382+
pfree(extval);
386383
}
387384
break;
388385
case BPCHAROID:

‎test/expected/decimal.out‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'binary');
3+
CREATE SERVER http_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
6+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS decimal_test');
7+
clickhouse_raw_query
8+
----------------------
9+
10+
(1 row)
11+
12+
SELECT clickhouse_raw_query('CREATE DATABASE decimal_test');
13+
clickhouse_raw_query
14+
----------------------
15+
16+
(1 row)
17+
18+
SELECT clickhouse_raw_query($$
19+
CREATE TABLE decimal_test.decimals (
20+
id Int32 NOT NULL,
21+
dec Decimal(8, 0) NOT NULL,
22+
dec32 Decimal32(4) NOT NULL,
23+
dec64 Decimal64(6) NOT NULL,
24+
dec128 Decimal128(8) NOT NULL
25+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
26+
$$);
27+
clickhouse_raw_query
28+
----------------------
29+
30+
(1 row)
31+
32+
CREATE SCHEMA dec_bin;
33+
CREATE SCHEMA dec_http;
34+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER binary_decimal_loopback INTO dec_bin;
35+
\d dec_bin.decimals
36+
Foreign table "dec_bin.decimals"
37+
Column | Type | Collation | Nullable | Default | FDW options
38+
--------+---------------+-----------+----------+---------+-------------
39+
id | integer | | not null | |
40+
dec | numeric(8,0) | | not null | |
41+
dec32 | numeric(9,4) | | not null | |
42+
dec64 | numeric(18,6) | | not null | |
43+
dec128 | numeric(38,8) | | not null | |
44+
Server: binary_decimal_loopback
45+
FDW options: (database 'decimal_test', table_name 'decimals', engine 'MergeTree')
46+
47+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER http_decimal_loopback INTO dec_http;
48+
\d dec_http.decimals
49+
Foreign table "dec_http.decimals"
50+
Column | Type | Collation | Nullable | Default | FDW options
51+
--------+---------------+-----------+----------+---------+-------------
52+
id | integer | | not null | |
53+
dec | numeric(8,0) | | not null | |
54+
dec32 | numeric(9,4) | | not null | |
55+
dec64 | numeric(18,6) | | not null | |
56+
dec128 | numeric(38,8) | | not null | |
57+
Server: http_decimal_loopback
58+
FDW options: (database 'decimal_test', table_name 'decimals', engine 'MergeTree')
59+
60+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
61+
INSERT INTO dec_bin.decimals (id, dec, dec32, dec64, dec128) VALUES
62+
(1, 42::NUMERIC, 98.6::NUMERIC, 102.4::NUMERIC, 1024.003::NUMERIC),
63+
(2, 9999, 9999.9999, 9999999.999999, 99999999999.99999999),
64+
(3, -9999, -9999.9999, -9999999.999999, -99999999999.99999999)
65+
;
66+
INSERT INTO dec_http.decimals VALUES
67+
(4, 1000000::NUMERIC, 10000::NUMERIC, 3000000000::NUMERIC, 400000000000::NUMERIC),
68+
(5, -1, -0.0001, -0.000001, -0.00000001),
69+
(6, 0, 0, 0, 0)
70+
;
71+
SELECT * FROM dec_bin.decimals ORDER BY id;
72+
id | dec | dec32 | dec64 | dec128
73+
----+---------+------------+-------------------+-----------------------
74+
1 | 42 | 98.6000 | 102.400000 | 1024.00300000
75+
2 | 9999 | 9999.9999 | 9999999.999999 | 99999999999.99999999
76+
3 | -9999 | -9999.9999 | -9999999.999999 | -99999999999.99999999
77+
4 | 1000000 | 10000.0000 | 3000000000.000000 | 400000000000.00000000
78+
5 | -1 | -0.0001 | -0.000001 | -0.00000001
79+
6 | 0 | 0.0000 | 0.000000 | 0.00000000
80+
(6 rows)
81+
82+
SELECT * FROM dec_http.decimals ORDER BY id;
83+
id | dec | dec32 | dec64 | dec128
84+
----+---------+------------+-------------------+-----------------------
85+
1 | 42 | 98.6000 | 102.400000 | 1024.00300000
86+
2 | 9999 | 9999.9999 | 9999999.999999 | 99999999999.99999999
87+
3 | -9999 | -9999.9999 | -9999999.999999 | -99999999999.99999999
88+
4 | 1000000 | 10000.0000 | 3000000000.000000 | 400000000000.00000000
89+
5 | -1 | -0.0001 | -0.000001 | -0.00000001
90+
6 | 0 | 0.0000 | 0.000000 | 0.00000000
91+
(6 rows)
92+
93+
SELECT clickhouse_raw_query('DROP DATABASE decimal_test');
94+
clickhouse_raw_query
95+
----------------------
96+
97+
(1 row)
98+
99+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
100+
DROP USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
101+
DROP SERVER binary_decimal_loopback CASCADE;
102+
NOTICE: drop cascades to foreign table dec_bin.decimals
103+
DROP SERVER http_decimal_loopback CASCADE;
104+
NOTICE: drop cascades to foreign table dec_http.decimals

‎test/sql/decimal.sql‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
SET datestyle = 'ISO';
2+
CREATE SERVER binary_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'binary');
3+
CREATE SERVER http_decimal_loopback FOREIGN DATA WRAPPER clickhouse_fdw OPTIONS(dbname 'decimal_test', driver 'http');
4+
CREATE USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
5+
CREATE USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
6+
7+
SELECT clickhouse_raw_query('DROP DATABASE IF EXISTS decimal_test');
8+
SELECT clickhouse_raw_query('CREATE DATABASE decimal_test');
9+
SELECT clickhouse_raw_query($$
10+
CREATE TABLE decimal_test.decimals (
11+
id Int32 NOT NULL,
12+
dec Decimal(8, 0) NOT NULL,
13+
dec32 Decimal32(4) NOT NULL,
14+
dec64 Decimal64(6) NOT NULL,
15+
dec128 Decimal128(8) NOT NULL
16+
) ENGINE = MergeTree PARTITION BY id ORDER BY (id);
17+
$$);
18+
19+
CREATE SCHEMA dec_bin;
20+
CREATE SCHEMA dec_http;
21+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER binary_decimal_loopback INTO dec_bin;
22+
\d dec_bin.decimals
23+
IMPORT FOREIGN SCHEMA "decimal_test" FROM SERVER http_decimal_loopback INTO dec_http;
24+
\d dec_http.decimals
25+
26+
-- Fails pending https://github.com/ClickHouse/clickhouse-cpp/issues/422
27+
INSERT INTO dec_bin.decimals (id, dec, dec32, dec64, dec128) VALUES
28+
(1, 42::NUMERIC, 98.6::NUMERIC, 102.4::NUMERIC, 1024.003::NUMERIC),
29+
(2, 9999, 9999.9999, 9999999.999999, 99999999999.99999999),
30+
(3, -9999, -9999.9999, -9999999.999999, -99999999999.99999999)
31+
;
32+
33+
INSERT INTO dec_http.decimals VALUES
34+
(4, 1000000::NUMERIC, 10000::NUMERIC, 3000000000::NUMERIC, 400000000000::NUMERIC),
35+
(5, -1, -0.0001, -0.000001, -0.00000001),
36+
(6, 0, 0, 0, 0)
37+
;
38+
39+
SELECT * FROM dec_bin.decimals ORDER BY id;
40+
SELECT * FROM dec_http.decimals ORDER BY id;
41+
42+
SELECT clickhouse_raw_query('DROP DATABASE decimal_test');
43+
DROP USER MAPPING FOR CURRENT_USER SERVER binary_decimal_loopback;
44+
DROP USER MAPPING FOR CURRENT_USER SERVER http_decimal_loopback;
45+
DROP SERVER binary_decimal_loopback CASCADE;
46+
DROP SERVER http_decimal_loopback CASCADE;

0 commit comments

Comments
 (0)