From 51defbe16caec174ecfc9cd8599d00e72bba0c7f Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 15 Dec 2019 16:34:17 -0500 Subject: [PATCH 01/16] Added support for Fetch One and returning dates as date string --- SeasClick.cpp | 69 ++++++++--------- php_SeasClick.h | 4 + tests/002.phpt | 2 +- tests/011.phpt | 200 ++++++++++++++++++++++++++++++++++++++++++++++++ tests/012.phpt | 81 ++++++++++++++++++++ typesToPhp.cpp | 181 +++++++++++++++++++++++++++++++------------ typesToPhp.hpp | 2 +- 7 files changed, 449 insertions(+), 90 deletions(-) create mode 100644 tests/011.phpt create mode 100755 tests/012.phpt diff --git a/SeasClick.cpp b/SeasClick.cpp index cfa0a22..9206201 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -64,9 +64,10 @@ ZEND_BEGIN_ARG_INFO_EX(SeasCilck_construct, 0, 0, 1) ZEND_ARG_INFO(0, connectParames) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(SeasCilck_select, 0, 0, 2) +ZEND_BEGIN_ARG_INFO_EX(SeasCilck_select, 0, 0, 3) ZEND_ARG_INFO(0, sql) ZEND_ARG_INFO(0, params) +ZEND_ARG_INFO(0, fetch_mode) ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(SeasCilck_insert, 0, 0, 3) @@ -98,6 +99,9 @@ const zend_function_entry SeasClick_methods[] = PHP_FE_END }; +#define REGISTER_SC_CLASS_CONST_LONG(const_name, value) \ + zend_declare_class_constant_long(SeasClick_ce, const_name, sizeof(const_name)-1, (zend_long)value); + /* {{{ PHP_MINIT_FUNCTION */ PHP_MINIT_FUNCTION(SeasClick) @@ -116,31 +120,11 @@ PHP_MINIT_FUNCTION(SeasClick) zend_declare_property_null(SeasClick_ce, "passwd", strlen("passwd"), ZEND_ACC_PROTECTED TSRMLS_CC); zend_declare_property_bool(SeasClick_ce, "compression", strlen("compression"), false, ZEND_ACC_PROTECTED TSRMLS_CC); - SeasClick_ce->ce_flags = ZEND_ACC_IMPLICIT_PUBLIC; - return SUCCESS; -} -/* }}} */ + REGISTER_SC_CLASS_CONST_LONG("FETCH_ONE", (zend_long)SC_FETCH_ONE); + REGISTER_SC_CLASS_CONST_LONG("FETCH_KEY_PAIR", (zend_long)SC_FETCH_KEY_PAIR); + REGISTER_SC_CLASS_CONST_LONG("DATE_AS_STRINGS", (zend_long)SC_FETCH_DATE_AS_STRINGS); -/* {{{ PHP_MSHUTDOWN_FUNCTION - */ -PHP_MSHUTDOWN_FUNCTION(SeasClick) -{ - return SUCCESS; -} -/* }}} */ - -/* {{{ PHP_RINIT_FUNCTION - */ -PHP_RINIT_FUNCTION(SeasClick) -{ - return SUCCESS; -} -/* }}} */ - -/* {{{ PHP_RSHUTDOWN_FUNCTION - */ -PHP_RSHUTDOWN_FUNCTION(SeasClick) -{ + SeasClick_ce->ce_flags = ZEND_ACC_IMPLICIT_PUBLIC; return SUCCESS; } /* }}} */ @@ -167,9 +151,9 @@ zend_module_entry SeasClick_module_entry = SEASCLICK_RES_NAME, SeasClick_functions, PHP_MINIT(SeasClick), - PHP_MSHUTDOWN(SeasClick), - PHP_RINIT(SeasClick), - PHP_RSHUTDOWN(SeasClick), + NULL, + NULL, + NULL, PHP_MINFO(SeasClick), PHP_SEASCLICK_VERSION, STANDARD_MODULE_PROPERTIES @@ -301,26 +285,29 @@ void getInsertSql(string *sql, char *table_name, zval *columns) *sql = "INSERT INTO " + (string)table_name + " ( " + fields_section.str() + " ) VALUES"; } -/* {{{ proto array select(string sql, array params) +/* {{{ proto array select(string sql, array params, int mode) */ PHP_METHOD(SEASCLICK_RES_NAME, select) { char *sql = NULL; size_t l_sql = 0; + zval *single_ret = NULL; zval* params = NULL; + zend_long fetch_mode = 0; #ifndef FAST_ZPP - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|z", &sql, &l_sql, ¶ms) == FAILURE) + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "s|zl", &sql, &l_sql, ¶ms, &fetch_mode) == FAILURE) { return; } #else #undef IS_UNDEF #define IS_UNDEF Z_EXPECTED_LONG - ZEND_PARSE_PARAMETERS_START(1, 2) + ZEND_PARSE_PARAMETERS_START(1, 3) Z_PARAM_STRING(sql, l_sql) Z_PARAM_OPTIONAL Z_PARAM_ARRAY(params) + Z_PARAM_LONG(fetch_mode) ZEND_PARSE_PARAMETERS_END(); #undef IS_UNDEF #define IS_UNDEF 0 @@ -352,10 +339,18 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) int key = Z_OBJ_HANDLE(*getThis()); Client *client = clientMap.at(key); - array_init(return_value); + if (!(fetch_mode & SC_FETCH_ONE)) { + array_init(return_value); + } + + client->Select(sql_s, [return_value, fetch_mode](const Block &block) { + if (fetch_mode & SC_FETCH_ONE) { + if (block.GetRowCount() > 0 && block.GetColumnCount() > 0) { + convertToZval(return_value, block[0], 0, "", 0, fetch_mode); + } + return; + } - client->Select(sql_s, [return_value](const Block& block) - { zval *return_tmp; for (size_t row = 0; row < block.GetRowCount(); ++row) { @@ -364,13 +359,11 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) for (size_t column = 0; column < block.GetColumnCount(); ++column) { string column_name = block.GetColumnName(column); - convertToZval(return_tmp, block[column], row, column_name, 0); + convertToZval(return_tmp, block[column], row, column_name, 0, fetch_mode); } add_next_index_zval(return_value, return_tmp); } - } - ); - + }); } catch (const std::exception& e) { diff --git a/php_SeasClick.h b/php_SeasClick.h index d0fe5fb..01a4a0d 100644 --- a/php_SeasClick.h +++ b/php_SeasClick.h @@ -49,6 +49,10 @@ typedef unsigned long ulong_t; #define SEASCLICK_G(v) (SeasClick_globals.v) #endif +#define SC_FETCH_ONE 1 +#define SC_FETCH_KEY_PAIR 2 +#define SC_FETCH_DATE_AS_STRINGS 4 + #define SEASCLICK_RES_NAME "SeasClick" #endif /* PHP_SEASCLICK_H */ diff --git a/tests/002.phpt b/tests/002.phpt index d69493a..402b94d 100644 --- a/tests/002.phpt +++ b/tests/002.phpt @@ -66,7 +66,7 @@ array(2) { } ["arraynull_c"]=> array(1) { - ["array"]=> + [0]=> string(6) "string" } } diff --git a/tests/011.phpt b/tests/011.phpt new file mode 100644 index 0000000..2646904 --- /dev/null +++ b/tests/011.phpt @@ -0,0 +1,200 @@ +--TEST-- +SeasClick Single Column Fetch +--SKIPIF-- + +--FILE-- + "clickhouse", + "port" => "9000", + "compression" => true, +]; + +$deleteTable = true; +$client = new SeasClick($config); +$client->execute('CREATE DATABASE IF NOT EXISTS test'); + +$client->execute("CREATE TABLE IF NOT EXISTS test.single_val_fetch ( + tuple_c Tuple(id UInt64, name String), + int64_c UInt64, + string_c String, + array_c Array(Int8), + arraynull_c Array(Nullable(String)), + enum8_c Enum8('One8' = 1, 'Two8' = 2), + enum16_c Enum16('One16' = 1, 'Two16' = 2), + fixedstring_c FixedString(50), + int8null_c Nullable(Int8), + stringnull_c Nullable(String), + enumnull_c Nullable(Enum8('One8' = 1, 'Two8' = 2)), + float32null_c Nullable(Float32), + uuidnull_c Nullable(UUID), + int8_c Int8, + int16_c Int16, + uint8_c UInt8, + uint16_c UInt16, + float32_c Float32, + float64_c Float64, + uuid_c UUID, + uuid2_c UUID, + date_c Date, + datetime_c DateTime +) ENGINE = Memory"); + +$data = [ + [ + 'tuple_c' => [1, 'one'], + 'int64_c' => 1, + 'string_c' => 'string_one', + 'array_c' => [1, 2, 3], + 'arraynull_c' => ['str_array'], + 'enum8_c' => 1, + 'enum16_c' => 'Two16', + 'fixedstring_c' => 'fixedstring_c1', + 'int8null_c' => 8, + 'stringnull_c' => 'string', + 'enumnull_c' => 'One8', + 'float32null_c' => 7.77, + 'uuidnull_c' => '31249a1b7b0542709f37c609b48a9bb2', + 'int8_c' => 8, + 'int16_c' => 16, + 'uint8_c' => 18, + 'uint16_c' => 20, + 'float32_c' => 32.32, + 'float64_c' => 64.64, + 'uuid_c' => '31249a1b-7b05-4270-9f37-c609b48a9bb2', + 'uuid2_c' => '31249a1b7b0542709f37c609b48a9bb2', + 'date_c' => 1548633600, + 'datetime_c' => 1548687925, + ], + [ + 'tuple_c' => [2, 'two'], + 'int64_c' => 2, + 'string_c' => 'string_two', + 'array_c' => [2, 3, 4], + 'arraynull_c' => [null], + 'enum8_c' => 'Two8', + 'enum16_c' => 2, + 'fixedstring_c' => 'fixedstring_c2', + 'int8null_c' => null, + 'stringnull_c' => null, + 'enumnull_c' => null, + 'float32null_c' => null, + 'uuidnull_c' => null, + 'int8_c' => 28, + 'int16_c' => 216, + 'uint8_c' => 218, + 'uint16_c' => 220, + 'float32_c' => 232.32, + 'float64_c' => 264.64, + 'uuid_c' => '31249a1b-7b05-4270-9f37-c609b48a9bb2', + 'uuid2_c' => null, + 'date_c' => 1548547200, + 'datetime_c' => 1548513600, + ], +]; + +$fields = array_keys(current($data)); + +$expected = $data; +$expected[0]['uuid_c'] = '31249a1b7b0542709f37c609b48a9bb2'; +$expected[0]['enum8_c'] = 'One8'; +$expected[1]['enum16_c'] = 'Two16'; +$expected[1]['uuid_c'] = '31249a1b7b0542709f37c609b48a9bb2'; +$expected[1]['arraynull_c'] = [null]; +$expected[1]['uuid2_c'] = '00000000000000000000000000000000'; + +$client->insert('test.single_val_fetch', $fields, [array_values($data[0]), array_values($data[1])]); + +foreach ($fields as $field) { + for ($i = 1; $i < 3; $i++) { + $result = $client->select("SELECT {$field} FROM test.single_val_fetch WHERE int64_c = {i}", ['i' => $i], SeasClick::FETCH_ONE); + $match = $result === $expected[$i-1][$field] ? 'OK' : 'FAIL'; + echo $field, ': ', var_export($result, true), ' - ', var_export($expected[$i - 1][$field], true), ' - ', $match, "\n"; + } +} + +$client->execute('DROP TABLE test.single_val_fetch'); +?> +--EXPECT-- +tuple_c: array ( + 0 => 1, + 1 => 'one', +) - array ( + 0 => 1, + 1 => 'one', +) - OK +tuple_c: array ( + 0 => 2, + 1 => 'two', +) - array ( + 0 => 2, + 1 => 'two', +) - OK +int64_c: 1 - 1 - OK +int64_c: 2 - 2 - OK +string_c: 'string_one' - 'string_one' - OK +string_c: 'string_two' - 'string_two' - OK +array_c: array ( + 0 => 1, + 1 => 2, + 2 => 3, +) - array ( + 0 => 1, + 1 => 2, + 2 => 3, +) - OK +array_c: array ( + 0 => 2, + 1 => 3, + 2 => 4, +) - array ( + 0 => 2, + 1 => 3, + 2 => 4, +) - OK +arraynull_c: array ( + 0 => 'str_array', +) - array ( + 0 => 'str_array', +) - OK +arraynull_c: array ( + 0 => NULL, +) - array ( + 0 => NULL, +) - OK +enum8_c: 'One8' - 'One8' - OK +enum8_c: 'Two8' - 'Two8' - OK +enum16_c: 'Two16' - 'Two16' - OK +enum16_c: 'Two16' - 'Two16' - OK +fixedstring_c: 'fixedstring_c1' - 'fixedstring_c1' - OK +fixedstring_c: 'fixedstring_c2' - 'fixedstring_c2' - OK +int8null_c: 8 - 8 - OK +int8null_c: NULL - NULL - OK +stringnull_c: 'string' - 'string' - OK +stringnull_c: NULL - NULL - OK +enumnull_c: 'One8' - 'One8' - OK +enumnull_c: NULL - NULL - OK +float32null_c: 7.77 - 7.77 - OK +float32null_c: NULL - NULL - OK +uuidnull_c: '31249a1b7b0542709f37c609b48a9bb2' - '31249a1b7b0542709f37c609b48a9bb2' - OK +uuidnull_c: NULL - NULL - OK +int8_c: 8 - 8 - OK +int8_c: 28 - 28 - OK +int16_c: 16 - 16 - OK +int16_c: 216 - 216 - OK +uint8_c: 18 - 18 - OK +uint8_c: 218 - 218 - OK +uint16_c: 20 - 20 - OK +uint16_c: 220 - 220 - OK +float32_c: 32.32 - 32.32 - OK +float32_c: 232.32 - 232.32 - OK +float64_c: 64.64 - 64.64 - OK +float64_c: 264.64 - 264.64 - OK +uuid_c: '31249a1b7b0542709f37c609b48a9bb2' - '31249a1b7b0542709f37c609b48a9bb2' - OK +uuid_c: '31249a1b7b0542709f37c609b48a9bb2' - '31249a1b7b0542709f37c609b48a9bb2' - OK +uuid2_c: '31249a1b7b0542709f37c609b48a9bb2' - '31249a1b7b0542709f37c609b48a9bb2' - OK +uuid2_c: '00000000000000000000000000000000' - '00000000000000000000000000000000' - OK +date_c: 1548633600 - 1548633600 - OK +date_c: 1548547200 - 1548547200 - OK +datetime_c: 1548687925 - 1548687925 - OK +datetime_c: 1548513600 - 1548513600 - OK diff --git a/tests/012.phpt b/tests/012.phpt new file mode 100755 index 0000000..eaad1da --- /dev/null +++ b/tests/012.phpt @@ -0,0 +1,81 @@ +--TEST-- +SeasClick Date Formatting +--SKIPIF-- + +--FILE-- + "clickhouse", + "port" => "9000", + "compression" => true, +]; + +$deleteTable = true; +$client = new SeasClick($config); +$client->execute('CREATE DATABASE IF NOT EXISTS test'); + +$client->execute("CREATE TABLE IF NOT EXISTS test.dates ( + date_c Date, + datetime_c DateTime +) ENGINE = Memory"); + +$data = [ + [ + 'date_c' => 1548633600, + 'datetime_c' => 1548687925, + ], + [ + 'date_c' => 1548547200, + 'datetime_c' => 1548513600, + ], +]; +$expected = [ + [ + 'date_c' => date('Y-m-d', 1548633600), + 'datetime_c' => date('Y-m-d H:i:s', 1548687925), + ], + [ + 'date_c' => date('Y-m-d', 1548547200), + 'datetime_c' => date('Y-m-d H:i:s', 1548513600), + ], +]; + +$fields = array_keys(current($data)); +$client->insert('test.dates', $fields, [array_values($data[0]), array_values($data[1])]); + +$res = $client->select("SELECT * FROM test.dates", [], SeasClick::DATE_AS_STRINGS); +var_dump($res); +if (array_diff_assoc($expected[0], $res[0]) || array_diff_assoc($expected[1], $res[1])) { + echo "FAIL\n"; +} else { + echo "OK\n"; +} + +$res = $client->select("SELECT date_c FROM test.dates WHERE datetime_c = 1548687925", [], SeasClick::FETCH_ONE|SeasClick::DATE_AS_STRINGS); +echo $res, ' = ', $expected[0]['date_c'], ' ', ($res === $expected[0]['date_c'] ? 'OK' : 'FAIL'), "\n"; + +$res = $client->select("SELECT datetime_c FROM test.dates WHERE datetime_c = 1548687925", [], SeasClick::FETCH_ONE|SeasClick::DATE_AS_STRINGS); +echo $res, ' = ', $expected[0]['datetime_c'], ' ' , ($res === $expected[0]['datetime_c'] ? 'OK' : 'FAIL'), "\n"; + +$client->execute('DROP TABLE test.dates'); +?> +--EXPECT-- +array(2) { + [0]=> + array(2) { + ["date_c"]=> + string(10) "2019-01-27" + ["datetime_c"]=> + string(19) "2019-01-28 10:05:25" + } + [1]=> + array(2) { + ["date_c"]=> + string(10) "2019-01-26" + ["datetime_c"]=> + string(19) "2019-01-26 09:40:00" + } +} +OK +2019-01-27 = 2019-01-27 OK +2019-01-28 10:05:25 = 2019-01-28 10:05:25 OK diff --git a/typesToPhp.cpp b/typesToPhp.cpp index 2823d36..25f5ac1 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -615,7 +615,28 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) throw std::runtime_error("insertColumn runtime error."); } -void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array) +#define SC_SINGLE_LONG() \ + if (fetch_mode & SC_FETCH_ONE) { \ + ZVAL_LONG(arr, (zend_ulong)col); \ + } else { \ + sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); \ + } + +#define SC_SINGLE_DOUBLE(val) \ + if (fetch_mode & SC_FETCH_ONE) { \ + ZVAL_DOUBLE(arr, val); \ + } else { \ + sc_add_assoc_double_ex(arr, column_name.c_str(), column_name.length(), val); \ + } + +#define SC_SINGLE_STRING(val, len) \ + if (fetch_mode & SC_FETCH_ONE) { \ + ZVAL_STRINGL(arr, val, len); \ + } else { \ + sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), val, len, 1); \ + } + +void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array, zend_long fetch_mode) { switch (columnRef->Type()->GetCode()) { @@ -628,7 +649,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -641,7 +662,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -654,7 +675,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -667,7 +688,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -681,7 +702,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -694,7 +715,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -707,7 +728,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -720,7 +741,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col); + SC_SINGLE_LONG(); } break; } @@ -738,7 +759,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), (char*)(first.str() + second.str()).c_str(), (first.str() + second.str()).length(), 1); + SC_SINGLE_STRING((char*)(first.str() + second.str()).c_str(), (first.str() + second.str()).length()); } break; } @@ -756,7 +777,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_double_ex(arr, column_name.c_str(), column_name.length(), d); + SC_SINGLE_DOUBLE(d); } break; } @@ -769,7 +790,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_double_ex(arr, column_name.c_str(), column_name.length(), (double)col); + SC_SINGLE_DOUBLE((double)col); } break; } @@ -783,7 +804,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), (char*)col.c_str(), col.length(), 1); + SC_SINGLE_STRING((char*)col.c_str(), col.length()); } break; } @@ -796,7 +817,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), (char*)col.c_str(), strlen((char*)col.c_str()), 1); + SC_SINGLE_STRING((char*)col.c_str(), strlen((char*)col.c_str())); } break; } @@ -806,11 +827,31 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column auto col = columnRef->As(); if (is_array) { - add_next_index_long(arr, (long)col->As()->At(row)); + if (fetch_mode & SC_FETCH_DATE_AS_STRINGS) { + char buffer[32]; + size_t l; + std::time_t t = (long)col->As()->At(row); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); + sc_add_next_index_stringl(arr, buffer, l, 1); + } else { + add_next_index_long(arr, (long)col->As()->At(row)); + } } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col->As()->At(row)); + if (fetch_mode & SC_FETCH_DATE_AS_STRINGS) { + char buffer[32]; + size_t l; + std::time_t t = (long)col->As()->At(row); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); + SC_SINGLE_STRING(buffer, l); + } else { + if (fetch_mode & SC_FETCH_ONE) { + ZVAL_LONG(arr, (long)col->As()->At(row)); + } else { + sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col->As()->At(row)); + } + } } break; } @@ -819,11 +860,31 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column auto col = columnRef->As(); if (is_array) { - add_next_index_long(arr, (long)col->As()->At(row)); + if (fetch_mode & SC_FETCH_DATE_AS_STRINGS) { + char buffer[16]; + size_t l; + std::time_t t = (long)col->As()->At(row); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); + sc_add_next_index_stringl(arr, buffer, l, 1); + } else { + add_next_index_long(arr, (long)col->As()->At(row)); + } } else { - sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col->As()->At(row)); + if (fetch_mode & SC_FETCH_DATE_AS_STRINGS) { + char buffer[16]; + size_t l; + std::time_t t = (long)col->As()->At(row); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); + SC_SINGLE_STRING(buffer, l); + } else { + if (fetch_mode & SC_FETCH_ONE) { + ZVAL_LONG(arr, (long)col->As()->At(row)); + } else { + sc_add_assoc_long_ex(arr, column_name.c_str(), column_name.length(), (zend_ulong)col->As()->At(row)); + } + } } break; } @@ -832,20 +893,28 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column { auto array = columnRef->As(); auto col = array->GetAsColumn(row); - zval *return_tmp; - SC_MAKE_STD_ZVAL(return_tmp); - array_init(return_tmp); - for (size_t i = 0; i < col->Size(); ++i) - { - convertToZval(return_tmp, col, i, "array", 1); - } - if (is_array) - { - add_next_index_zval(arr, return_tmp); - } - else - { - sc_add_assoc_zval_ex(arr, column_name.c_str(), column_name.length(), return_tmp); + if (fetch_mode & SC_FETCH_ONE) { + array_init(arr); + for (size_t i = 0; i < col->Size(); ++i) + { + convertToZval(arr, col, i, "array", 1, 0); + } + } else { + zval *return_tmp; + SC_MAKE_STD_ZVAL(return_tmp); + array_init(return_tmp); + for (size_t i = 0; i < col->Size(); ++i) + { + convertToZval(return_tmp, col, i, "array", 1, 0); + } + if (is_array) + { + add_next_index_zval(arr, return_tmp); + } + else + { + sc_add_assoc_zval_ex(arr, column_name.c_str(), column_name.length(), return_tmp); + } } break; } @@ -859,7 +928,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), (char*)array->NameAt(row).c_str(), array->NameAt(row).length(), 1); + SC_SINGLE_STRING((char*)array->NameAt(row).c_str(), array->NameAt(row).length()); } break; } @@ -872,7 +941,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), (char*)array->NameAt(row).c_str(), array->NameAt(row).length(), 1); + SC_SINGLE_STRING((char*)array->NameAt(row).c_str(), array->NameAt(row).length()); } break; } @@ -888,12 +957,16 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } else { - sc_add_assoc_null_ex(arr, column_name.c_str(), column_name.length()); + if (fetch_mode & SC_FETCH_ONE) { + ZVAL_NULL(arr); + } else { + sc_add_assoc_null_ex(arr, column_name.c_str(), column_name.length()); + } } } else { - convertToZval(arr, nullable->Nested(), row, column_name, 0); + convertToZval(arr, nullable->Nested(), row, column_name, is_array, fetch_mode); } break; } @@ -901,20 +974,28 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column case Type::Code::Tuple: { auto tuple = columnRef->As(); - zval *return_tmp; - SC_MAKE_STD_ZVAL(return_tmp); - array_init(return_tmp); - for (size_t i = 0; i < tuple->tupleSize(); ++i) - { - convertToZval(return_tmp, (*tuple)[i], row, "tuple", 1); - } - if (is_array) - { - add_next_index_zval(arr, return_tmp); - } - else - { - sc_add_assoc_zval_ex(arr, column_name.c_str(), column_name.length(), return_tmp); + if (fetch_mode & SC_FETCH_ONE) { + array_init(arr); + for (size_t i = 0; i < tuple->tupleSize(); ++i) + { + convertToZval(arr, (*tuple)[i], row, "tuple", 1, 0); + } + } else { + zval *return_tmp; + SC_MAKE_STD_ZVAL(return_tmp); + array_init(return_tmp); + for (size_t i = 0; i < tuple->tupleSize(); ++i) + { + convertToZval(return_tmp, (*tuple)[i], row, "tuple", 1, 0); + } + if (is_array) + { + add_next_index_zval(arr, return_tmp); + } + else + { + sc_add_assoc_zval_ex(arr, column_name.c_str(), column_name.length(), return_tmp); + } } break; } diff --git a/typesToPhp.hpp b/typesToPhp.hpp index a08b67f..0a5f51b 100644 --- a/typesToPhp.hpp +++ b/typesToPhp.hpp @@ -22,7 +22,7 @@ ColumnRef createColumn(TypeRef type); ColumnRef insertColumn(TypeRef type, zval *value_zval); -void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array); +void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array, zend_long fetch_mode); void zvalToBlock(Block& blockDes, Block& blockSrc, zend_ulong num_key, zval *value_zval); From 1e8cfd0dfa94de8df77976756bedb6e91f9aaa8e Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 15 Dec 2019 17:28:23 -0500 Subject: [PATCH 02/16] Added support for fetching values of a single column as an array --- SeasClick.cpp | 13 ++++++++++--- php_SeasClick.h | 1 + tests/011.phpt | 2 +- 3 files changed, 12 insertions(+), 4 deletions(-) diff --git a/SeasClick.cpp b/SeasClick.cpp index 9206201..062c701 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -123,6 +123,7 @@ PHP_MINIT_FUNCTION(SeasClick) REGISTER_SC_CLASS_CONST_LONG("FETCH_ONE", (zend_long)SC_FETCH_ONE); REGISTER_SC_CLASS_CONST_LONG("FETCH_KEY_PAIR", (zend_long)SC_FETCH_KEY_PAIR); REGISTER_SC_CLASS_CONST_LONG("DATE_AS_STRINGS", (zend_long)SC_FETCH_DATE_AS_STRINGS); + REGISTER_SC_CLASS_CONST_LONG("FETCH_COLUMN", (zend_long)SC_FETCH_COLUMN); SeasClick_ce->ce_flags = ZEND_ACC_IMPLICIT_PUBLIC; return SUCCESS; @@ -291,7 +292,6 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) { char *sql = NULL; size_t l_sql = 0; - zval *single_ret = NULL; zval* params = NULL; zend_long fetch_mode = 0; @@ -355,11 +355,18 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) for (size_t row = 0; row < block.GetRowCount(); ++row) { SC_MAKE_STD_ZVAL(return_tmp); - array_init(return_tmp); + if (!(fetch_mode & SC_FETCH_COLUMN)) { + array_init(return_tmp); + } for (size_t column = 0; column < block.GetColumnCount(); ++column) { string column_name = block.GetColumnName(column); - convertToZval(return_tmp, block[column], row, column_name, 0, fetch_mode); + if (fetch_mode & SC_FETCH_COLUMN) { + convertToZval(return_tmp, block[0], row, "", 0, fetch_mode|SC_FETCH_ONE); + break; + } else { + convertToZval(return_tmp, block[column], row, column_name, 0, fetch_mode); + } } add_next_index_zval(return_value, return_tmp); } diff --git a/php_SeasClick.h b/php_SeasClick.h index 01a4a0d..bd016b5 100644 --- a/php_SeasClick.h +++ b/php_SeasClick.h @@ -52,6 +52,7 @@ typedef unsigned long ulong_t; #define SC_FETCH_ONE 1 #define SC_FETCH_KEY_PAIR 2 #define SC_FETCH_DATE_AS_STRINGS 4 +#define SC_FETCH_COLUMN 8 #define SEASCLICK_RES_NAME "SeasClick" diff --git a/tests/011.phpt b/tests/011.phpt index 2646904..08c532b 100644 --- a/tests/011.phpt +++ b/tests/011.phpt @@ -1,5 +1,5 @@ --TEST-- -SeasClick Single Column Fetch +SeasClick Single Value Fetch --SKIPIF-- --FILE-- From e8fbbcaf14aad12ddf8ecd0282da503be88da6a4 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 15 Dec 2019 17:56:03 -0500 Subject: [PATCH 03/16] Make fetch key pair mode work --- SeasClick.cpp | 22 ++++ tests/013.phpt | 329 +++++++++++++++++++++++++++++++++++++++++++++++++ tests/014.phpt | 250 +++++++++++++++++++++++++++++++++++++ 3 files changed, 601 insertions(+) create mode 100755 tests/013.phpt create mode 100755 tests/014.phpt diff --git a/SeasClick.cpp b/SeasClick.cpp index 062c701..3c93e16 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -354,10 +354,32 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) zval *return_tmp; for (size_t row = 0; row < block.GetRowCount(); ++row) { + if (fetch_mode & SC_FETCH_KEY_PAIR) { + if (block.GetColumnCount() < 2) { + throw std::runtime_error("Key pair mode requires at least 2 columns to be present"); + } + zval *col1, *col2; + SC_MAKE_STD_ZVAL(col1); + SC_MAKE_STD_ZVAL(col2); + + convertToZval(col1, block[0], row, "", 0, fetch_mode|SC_FETCH_ONE); + convertToZval(col2, block[1], row, "", 0, fetch_mode|SC_FETCH_ONE); + + if (Z_TYPE_P(col1) == IS_LONG) { + zend_hash_index_update(Z_ARRVAL_P(return_value), Z_LVAL_P(col1), col2); + } else { + convert_to_string(col1); + zend_symtable_update(Z_ARRVAL_P(return_value), Z_STR_P(col1), col2); + } + zval_ptr_dtor(col1); + continue; + } + SC_MAKE_STD_ZVAL(return_tmp); if (!(fetch_mode & SC_FETCH_COLUMN)) { array_init(return_tmp); } + for (size_t column = 0; column < block.GetColumnCount(); ++column) { string column_name = block.GetColumnName(column); diff --git a/tests/013.phpt b/tests/013.phpt new file mode 100755 index 0000000..ea848d0 --- /dev/null +++ b/tests/013.phpt @@ -0,0 +1,329 @@ +--TEST-- +SeasClick Single Column Fetch +--SKIPIF-- + +--FILE-- + "clickhouse", + "port" => "9000", + "compression" => true, +]; + +$deleteTable = true; +$client = new SeasClick($config); +$client->execute('CREATE DATABASE IF NOT EXISTS test'); + +$client->execute("CREATE TABLE IF NOT EXISTS test.single_val_fetch ( + tuple_c Tuple(id UInt64, name String), + int64_c UInt64, + string_c String, + array_c Array(Int8), + arraynull_c Array(Nullable(String)), + enum8_c Enum8('One8' = 1, 'Two8' = 2), + enum16_c Enum16('One16' = 1, 'Two16' = 2), + fixedstring_c FixedString(50), + int8null_c Nullable(Int8), + stringnull_c Nullable(String), + enumnull_c Nullable(Enum8('One8' = 1, 'Two8' = 2)), + float32null_c Nullable(Float32), + uuidnull_c Nullable(UUID), + int8_c Int8, + int16_c Int16, + uint8_c UInt8, + uint16_c UInt16, + float32_c Float32, + float64_c Float64, + uuid_c UUID, + uuid2_c UUID, + date_c Date, + datetime_c DateTime +) ENGINE = Memory"); + +$data = [ + [ + 'tuple_c' => [1, 'one'], + 'int64_c' => 1, + 'string_c' => 'string_one', + 'array_c' => [1, 2, 3], + 'arraynull_c' => ['str_array'], + 'enum8_c' => 1, + 'enum16_c' => 'Two16', + 'fixedstring_c' => 'fixedstring_c1', + 'int8null_c' => 8, + 'stringnull_c' => 'string', + 'enumnull_c' => 'One8', + 'float32null_c' => 7.77, + 'uuidnull_c' => '31249a1b7b0542709f37c609b48a9bb2', + 'int8_c' => 8, + 'int16_c' => 16, + 'uint8_c' => 18, + 'uint16_c' => 20, + 'float32_c' => 32.32, + 'float64_c' => 64.64, + 'uuid_c' => '31249a1b-7b05-4270-9f37-c609b48a9bb2', + 'uuid2_c' => '31249a1b7b0542709f37c609b48a9bb2', + 'date_c' => 1548633600, + 'datetime_c' => 1548687925, + ], + [ + 'tuple_c' => [2, 'two'], + 'int64_c' => 2, + 'string_c' => 'string_two', + 'array_c' => [2, 3, 4], + 'arraynull_c' => [null], + 'enum8_c' => 'Two8', + 'enum16_c' => 2, + 'fixedstring_c' => 'fixedstring_c2', + 'int8null_c' => null, + 'stringnull_c' => null, + 'enumnull_c' => null, + 'float32null_c' => null, + 'uuidnull_c' => null, + 'int8_c' => 28, + 'int16_c' => 216, + 'uint8_c' => 218, + 'uint16_c' => 220, + 'float32_c' => 232.32, + 'float64_c' => 264.64, + 'uuid_c' => '31249a1b-7b05-4270-9f37-c609b48a9bb2', + 'uuid2_c' => null, + 'date_c' => 1548547200, + 'datetime_c' => 1548513600, + ], +]; + +$fields = array_keys(current($data)); + +$expected = $data; +$expected[0]['uuid_c'] = '31249a1b7b0542709f37c609b48a9bb2'; +$expected[0]['enum8_c'] = 'One8'; +$expected[1]['enum16_c'] = 'Two16'; +$expected[1]['uuid_c'] = '31249a1b7b0542709f37c609b48a9bb2'; +$expected[1]['arraynull_c'] = [null]; +$expected[1]['uuid2_c'] = '00000000000000000000000000000000'; + +$client->insert('test.single_val_fetch', $fields, [array_values($data[0]), array_values($data[1])]); + +foreach ($fields as $field) { + $result = $client->select("SELECT {$field} FROM test.single_val_fetch ORDER BY int64_c ASC", [], SeasClick::FETCH_COLUMN); + + $res = var_export($result, true); + $exp = var_export([$expected[0][$field], $expected[1][$field]], true); + $match = $res === $exp ? 'OK' : 'FAIL'; + + echo $field, ': ', $res , ' - ', $exp , ' - ', $match, "\n"; +} + +$client->execute('DROP TABLE test.single_val_fetch'); +?> +--EXPECT-- +tuple_c: array ( + 0 => + array ( + 0 => 1, + 1 => 'one', + ), + 1 => + array ( + 0 => 2, + 1 => 'two', + ), +) - array ( + 0 => + array ( + 0 => 1, + 1 => 'one', + ), + 1 => + array ( + 0 => 2, + 1 => 'two', + ), +) - OK +int64_c: array ( + 0 => 1, + 1 => 2, +) - array ( + 0 => 1, + 1 => 2, +) - OK +string_c: array ( + 0 => 'string_one', + 1 => 'string_two', +) - array ( + 0 => 'string_one', + 1 => 'string_two', +) - OK +array_c: array ( + 0 => + array ( + 0 => 1, + 1 => 2, + 2 => 3, + ), + 1 => + array ( + 0 => 2, + 1 => 3, + 2 => 4, + ), +) - array ( + 0 => + array ( + 0 => 1, + 1 => 2, + 2 => 3, + ), + 1 => + array ( + 0 => 2, + 1 => 3, + 2 => 4, + ), +) - OK +arraynull_c: array ( + 0 => + array ( + 0 => 'str_array', + ), + 1 => + array ( + 0 => NULL, + ), +) - array ( + 0 => + array ( + 0 => 'str_array', + ), + 1 => + array ( + 0 => NULL, + ), +) - OK +enum8_c: array ( + 0 => 'One8', + 1 => 'Two8', +) - array ( + 0 => 'One8', + 1 => 'Two8', +) - OK +enum16_c: array ( + 0 => 'Two16', + 1 => 'Two16', +) - array ( + 0 => 'Two16', + 1 => 'Two16', +) - OK +fixedstring_c: array ( + 0 => 'fixedstring_c1', + 1 => 'fixedstring_c2', +) - array ( + 0 => 'fixedstring_c1', + 1 => 'fixedstring_c2', +) - OK +int8null_c: array ( + 0 => 8, + 1 => NULL, +) - array ( + 0 => 8, + 1 => NULL, +) - OK +stringnull_c: array ( + 0 => 'string', + 1 => NULL, +) - array ( + 0 => 'string', + 1 => NULL, +) - OK +enumnull_c: array ( + 0 => 'One8', + 1 => NULL, +) - array ( + 0 => 'One8', + 1 => NULL, +) - OK +float32null_c: array ( + 0 => 7.77, + 1 => NULL, +) - array ( + 0 => 7.77, + 1 => NULL, +) - OK +uuidnull_c: array ( + 0 => '31249a1b7b0542709f37c609b48a9bb2', + 1 => NULL, +) - array ( + 0 => '31249a1b7b0542709f37c609b48a9bb2', + 1 => NULL, +) - OK +int8_c: array ( + 0 => 8, + 1 => 28, +) - array ( + 0 => 8, + 1 => 28, +) - OK +int16_c: array ( + 0 => 16, + 1 => 216, +) - array ( + 0 => 16, + 1 => 216, +) - OK +uint8_c: array ( + 0 => 18, + 1 => 218, +) - array ( + 0 => 18, + 1 => 218, +) - OK +uint16_c: array ( + 0 => 20, + 1 => 220, +) - array ( + 0 => 20, + 1 => 220, +) - OK +float32_c: array ( + 0 => 32.32, + 1 => 232.32, +) - array ( + 0 => 32.32, + 1 => 232.32, +) - OK +float64_c: array ( + 0 => 64.64, + 1 => 264.64, +) - array ( + 0 => 64.64, + 1 => 264.64, +) - OK +uuid_c: array ( + 0 => '31249a1b7b0542709f37c609b48a9bb2', + 1 => '31249a1b7b0542709f37c609b48a9bb2', +) - array ( + 0 => '31249a1b7b0542709f37c609b48a9bb2', + 1 => '31249a1b7b0542709f37c609b48a9bb2', +) - OK +uuid2_c: array ( + 0 => '31249a1b7b0542709f37c609b48a9bb2', + 1 => '00000000000000000000000000000000', +) - array ( + 0 => '31249a1b7b0542709f37c609b48a9bb2', + 1 => '00000000000000000000000000000000', +) - OK +date_c: array ( + 0 => 1548633600, + 1 => 1548547200, +) - array ( + 0 => 1548633600, + 1 => 1548547200, +) - OK +datetime_c: array ( + 0 => 1548687925, + 1 => 1548513600, +) - array ( + 0 => 1548687925, + 1 => 1548513600, +) - OK \ No newline at end of file diff --git a/tests/014.phpt b/tests/014.phpt new file mode 100755 index 0000000..78c21cc --- /dev/null +++ b/tests/014.phpt @@ -0,0 +1,250 @@ +--TEST-- +SeasClick Fetch Key Pair +--SKIPIF-- + +--FILE-- + "clickhouse", + "port" => "9000", + "compression" => true, +]; + +$deleteTable = true; +$client = new SeasClick($config); +$client->execute('CREATE DATABASE IF NOT EXISTS test'); + +$client->execute("CREATE TABLE IF NOT EXISTS test.single_val_fetch ( + tuple_c Tuple(id UInt64, name String), + int64_c UInt64, + string_c String, + array_c Array(Int8), + arraynull_c Array(Nullable(String)), + enum8_c Enum8('One8' = 1, 'Two8' = 2), + enum16_c Enum16('One16' = 1, 'Two16' = 2), + fixedstring_c FixedString(50), + int8null_c Nullable(Int8), + stringnull_c Nullable(String), + enumnull_c Nullable(Enum8('One8' = 1, 'Two8' = 2)), + float32null_c Nullable(Float32), + uuidnull_c Nullable(UUID), + int8_c Int8, + int16_c Int16, + uint8_c UInt8, + uint16_c UInt16, + float32_c Float32, + float64_c Float64, + uuid_c UUID, + uuid2_c UUID, + date_c Date, + datetime_c DateTime +) ENGINE = Memory"); + +$data = [ + [ + 'int64_c' => 1, + 'string_c' => 'string_one', + 'enum8_c' => 1, + 'enum16_c' => 'Two16', + 'fixedstring_c' => 'fixedstring_c1', + 'int8null_c' => 8, + 'stringnull_c' => 'string', + 'enumnull_c' => 'One8', + 'float32null_c' => 7.77, + 'uuidnull_c' => '31249a1b7b0542709f37c609b48a9bb2', + 'int8_c' => 8, + 'int16_c' => 16, + 'uint8_c' => 18, + 'uint16_c' => 20, + 'float32_c' => 32.32, + 'float64_c' => 64.64, + 'uuid_c' => '31249a1b-7b05-4270-9f37-c609b48a9bb2', + 'uuid2_c' => '31249a1b7b0542709f37c609b48a9bb2', + 'date_c' => 1548633600, + 'datetime_c' => 1548687925, + ], + [ + 'int64_c' => 2, + 'string_c' => 'string_two', + 'enum8_c' => 'Two8', + 'enum16_c' => 2, + 'fixedstring_c' => 'fixedstring_c2', + 'int8null_c' => null, + 'stringnull_c' => null, + 'enumnull_c' => null, + 'float32null_c' => null, + 'uuidnull_c' => null, + 'int8_c' => 28, + 'int16_c' => 216, + 'uint8_c' => 218, + 'uint16_c' => 220, + 'float32_c' => 232.32, + 'float64_c' => 264.64, + 'uuid_c' => '31249a1b-7b05-4270-9f37-c609b48a9bb2', + 'uuid2_c' => null, + 'date_c' => 1548547200, + 'datetime_c' => 1548513600, + ], +]; + +$fields = array_keys(current($data)); + +$expected = $data; +$expected[0]['uuid_c'] = '31249a1b7b0542709f37c609b48a9bb2'; +$expected[0]['enum8_c'] = 'One8'; +$expected[1]['enum16_c'] = 'Two16'; +$expected[1]['uuid_c'] = '31249a1b7b0542709f37c609b48a9bb2'; +$expected[1]['arraynull_c'] = [null]; +$expected[1]['uuid2_c'] = '00000000000000000000000000000000'; + +$client->insert('test.single_val_fetch', $fields, [array_values($data[0]), array_values($data[1])]); + +foreach ($fields as $field) { + $result = $client->select("SELECT {$field}, {$field} FROM test.single_val_fetch ORDER BY int64_c ASC", [], SeasClick::FETCH_KEY_PAIR); + + $res = var_export($result, true); + $exp = var_export([(string)$expected[0][$field] => $expected[0][$field], (string)$expected[1][$field] => $expected[1][$field]], true); + $match = $res === $exp ? 'OK' : 'FAIL'; + + echo $field, ': ', $res , ' - ', $exp , ' - ', $match, "\n"; +} + +$client->execute('DROP TABLE test.single_val_fetch'); +?> +--EXPECT-- +int64_c: array ( + 1 => 1, + 2 => 2, +) - array ( + 1 => 1, + 2 => 2, +) - OK +string_c: array ( + 'string_one' => 'string_one', + 'string_two' => 'string_two', +) - array ( + 'string_one' => 'string_one', + 'string_two' => 'string_two', +) - OK +enum8_c: array ( + 'One8' => 'One8', + 'Two8' => 'Two8', +) - array ( + 'One8' => 'One8', + 'Two8' => 'Two8', +) - OK +enum16_c: array ( + 'Two16' => 'Two16', +) - array ( + 'Two16' => 'Two16', +) - OK +fixedstring_c: array ( + 'fixedstring_c1' => 'fixedstring_c1', + 'fixedstring_c2' => 'fixedstring_c2', +) - array ( + 'fixedstring_c1' => 'fixedstring_c1', + 'fixedstring_c2' => 'fixedstring_c2', +) - OK +int8null_c: array ( + 8 => 8, + '' => NULL, +) - array ( + 8 => 8, + '' => NULL, +) - OK +stringnull_c: array ( + 'string' => 'string', + '' => NULL, +) - array ( + 'string' => 'string', + '' => NULL, +) - OK +enumnull_c: array ( + 'One8' => 'One8', + '' => NULL, +) - array ( + 'One8' => 'One8', + '' => NULL, +) - OK +float32null_c: array ( + '7.77' => 7.77, + '' => NULL, +) - array ( + '7.77' => 7.77, + '' => NULL, +) - OK +uuidnull_c: array ( + '31249a1b7b0542709f37c609b48a9bb2' => '31249a1b7b0542709f37c609b48a9bb2', + '' => NULL, +) - array ( + '31249a1b7b0542709f37c609b48a9bb2' => '31249a1b7b0542709f37c609b48a9bb2', + '' => NULL, +) - OK +int8_c: array ( + 8 => 8, + 28 => 28, +) - array ( + 8 => 8, + 28 => 28, +) - OK +int16_c: array ( + 16 => 16, + 216 => 216, +) - array ( + 16 => 16, + 216 => 216, +) - OK +uint8_c: array ( + 18 => 18, + 218 => 218, +) - array ( + 18 => 18, + 218 => 218, +) - OK +uint16_c: array ( + 20 => 20, + 220 => 220, +) - array ( + 20 => 20, + 220 => 220, +) - OK +float32_c: array ( + '32.32' => 32.32, + '232.32' => 232.32, +) - array ( + '32.32' => 32.32, + '232.32' => 232.32, +) - OK +float64_c: array ( + '64.64' => 64.64, + '264.64' => 264.64, +) - array ( + '64.64' => 64.64, + '264.64' => 264.64, +) - OK +uuid_c: array ( + '31249a1b7b0542709f37c609b48a9bb2' => '31249a1b7b0542709f37c609b48a9bb2', +) - array ( + '31249a1b7b0542709f37c609b48a9bb2' => '31249a1b7b0542709f37c609b48a9bb2', +) - OK +uuid2_c: array ( + '31249a1b7b0542709f37c609b48a9bb2' => '31249a1b7b0542709f37c609b48a9bb2', + '00000000000000000000000000000000' => '00000000000000000000000000000000', +) - array ( + '31249a1b7b0542709f37c609b48a9bb2' => '31249a1b7b0542709f37c609b48a9bb2', + '00000000000000000000000000000000' => '00000000000000000000000000000000', +) - OK +date_c: array ( + 1548633600 => 1548633600, + 1548547200 => 1548547200, +) - array ( + 1548633600 => 1548633600, + 1548547200 => 1548547200, +) - OK +datetime_c: array ( + 1548687925 => 1548687925, + 1548513600 => 1548513600, +) - array ( + 1548687925 => 1548687925, + 1548513600 => 1548513600, +) - OK \ No newline at end of file From 67b05550e449b0f9742d78478aa1c400bc2eb079 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 15 Dec 2019 17:57:30 -0500 Subject: [PATCH 04/16] update credit --- SeasClick.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SeasClick.cpp b/SeasClick.cpp index 3c93e16..e6e33ce 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -137,7 +137,7 @@ PHP_MINFO_FUNCTION(SeasClick) php_info_print_table_start(); php_info_print_table_header(2, "SeasClick support", "enabled"); php_info_print_table_row(2, "Version", PHP_SEASCLICK_VERSION); - php_info_print_table_row(2, "Author", "SeasX Group[email: ahhhh.wang@gmail.com]"); + php_info_print_table_row(2, "Author", "SeasX Group[email: ahhhh.wang@gmail.com], Ilia Alshanetsky"); php_info_print_table_end(); DISPLAY_INI_ENTRIES(); From b76bdaabf8e4da5a5b4b9e294c3b2dd095f428cf Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Sun, 15 Dec 2019 18:25:23 -0500 Subject: [PATCH 05/16] Use SeasClickException exception class --- SeasClick.cpp | 19 ++++++++++++------- tests/015.phpt | 22 ++++++++++++++++++++++ 2 files changed, 34 insertions(+), 7 deletions(-) create mode 100755 tests/015.phpt diff --git a/SeasClick.cpp b/SeasClick.cpp index e6e33ce..8406e98 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -40,7 +40,7 @@ extern "C" { using namespace clickhouse; using namespace std; -zend_class_entry *SeasClick_ce; +zend_class_entry *SeasClick_ce, *SeasClickException_ce; map clientMap; #ifdef COMPILE_DL_SEASCLICK @@ -106,13 +106,18 @@ const zend_function_entry SeasClick_methods[] = */ PHP_MINIT_FUNCTION(SeasClick) { - zend_class_entry SeasClick; + zend_class_entry SeasClick, SeasClickException; INIT_CLASS_ENTRY(SeasClick, SEASCLICK_RES_NAME, SeasClick_methods); + INIT_CLASS_ENTRY(SeasClickException, "SeasClickException", NULL); + #if PHP_VERSION_ID >= 70000 SeasClick_ce = zend_register_internal_class_ex(&SeasClick, NULL); + SeasClickException_ce = zend_register_internal_class_ex(&SeasClickException, zend_ce_exception); #else SeasClick_ce = zend_register_internal_class_ex(&SeasClick, NULL, NULL TSRMLS_CC); + SeasClickException_ce = zend_register_internal_class_ex(&SeasClickException, zend_exception_get_default(TSRMLS_C), NULL TSRMLS_CC); #endif + zend_declare_property_stringl(SeasClick_ce, "host", strlen("host"), "127.0.0.1", sizeof("127.0.0.1") - 1, ZEND_ACC_PROTECTED TSRMLS_CC); zend_declare_property_long(SeasClick_ce, "port", strlen("port"), 9000, ZEND_ACC_PROTECTED TSRMLS_CC); zend_declare_property_stringl(SeasClick_ce, "database", strlen("database"), "default", sizeof("default") - 1, ZEND_ACC_PROTECTED TSRMLS_CC); @@ -249,7 +254,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, __construct) } catch (const std::exception& e) { - sc_zend_throw_exception(NULL, e.what(), 0 TSRMLS_CC); + sc_zend_throw_exception(SeasClickException_ce, e.what(), 0 TSRMLS_CC); } RETURN_TRUE; @@ -396,7 +401,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) } catch (const std::exception& e) { - sc_zend_throw_exception(NULL, e.what(), 0 TSRMLS_CC); + sc_zend_throw_exception(SeasClickException_ce, e.what(), 0 TSRMLS_CC); } } /* }}} */ @@ -498,7 +503,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, insert) } catch (const std::exception& e) { - sc_zend_throw_exception(NULL, e.what(), 0 TSRMLS_CC); + sc_zend_throw_exception(SeasClickException_ce, e.what(), 0 TSRMLS_CC); } RETURN_TRUE; } @@ -560,7 +565,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, execute) } catch (const std::exception& e) { - sc_zend_throw_exception(NULL, e.what(), 0 TSRMLS_CC); + sc_zend_throw_exception(SeasClickException_ce, e.what(), 0 TSRMLS_CC); } RETURN_TRUE; } @@ -580,7 +585,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, __destruct) } catch (const std::exception& e) { - sc_zend_throw_exception(NULL, e.what(), 0 TSRMLS_CC); + sc_zend_throw_exception(SeasClickException_ce, e.what(), 0 TSRMLS_CC); } RETURN_TRUE; } diff --git a/tests/015.phpt b/tests/015.phpt new file mode 100755 index 0000000..9eb7f8e --- /dev/null +++ b/tests/015.phpt @@ -0,0 +1,22 @@ +--TEST-- +SeasClick Exception test +--SKIPIF-- + +--FILE-- + "clickhouse", + "port" => "9000", + "compression" => true, +]; + +$deleteTable = true; +$client = new SeasClick($config); +try { + $client->execute('FOO'); +} catch (SeasClickException $e) { + var_dump(get_class($e)); +} +?> +--EXPECT-- +string(18) "SeasClickException" From e62cee54f97d2ff44c84e6cb69e31f8e5990ed8b Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Mon, 6 Apr 2020 08:47:04 -0400 Subject: [PATCH 06/16] PHP5 compat --- SeasClick.cpp | 2 +- php7_wrapper.h | 9 +++++++++ typesToPhp.cpp | 2 +- typesToPhp.hpp | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/SeasClick.cpp b/SeasClick.cpp index 8406e98..e8aa497 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -371,7 +371,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, select) convertToZval(col2, block[1], row, "", 0, fetch_mode|SC_FETCH_ONE); if (Z_TYPE_P(col1) == IS_LONG) { - zend_hash_index_update(Z_ARRVAL_P(return_value), Z_LVAL_P(col1), col2); + sc_zend_hash_index_update(Z_ARRVAL_P(return_value), Z_LVAL_P(col1), col2); } else { convert_to_string(col1); zend_symtable_update(Z_ARRVAL_P(return_value), Z_STR_P(col1), col2); diff --git a/php7_wrapper.h b/php7_wrapper.h index 2306ff8..fe0a03d 100644 --- a/php7_wrapper.h +++ b/php7_wrapper.h @@ -23,6 +23,8 @@ // PHP7+ #if PHP_MAJOR_VERSION < 7 +typedef long zend_long; + #if PHP_VERSION_ID < 50500 #define sc_zend_throw_exception(a, b, c) zend_throw_exception(a, (char *)b, c) #else @@ -34,6 +36,12 @@ #define SC_RETURN_STRINGL(k, l) RETURN_STRINGL(k, l, 1) #define sc_zval_ptr_dtor zval_ptr_dtor #define sc_zval_add_ref(a) zval_add_ref(&a) + +static inline int sc_zend_hash_index_update(HashTable *ht, ulong h, void *pData) +{ + return zend_hash_index_update(ht, h, pData, sizeof(zval *), NULL); +} + static inline int sc_add_assoc_long_ex(zval *arg, const char *key, size_t key_len, long value) { return add_assoc_long_ex(arg, key, key_len + 1, value); @@ -107,6 +115,7 @@ static inline zval *sc_zend_hash_index_find(HashTable *ht, ulong h) // PHP5 #define sc_zend_throw_exception zend_throw_exception +#define sc_zend_hash_index_update zend_hash_index_update #define sc_zend_hash_find zend_hash_str_find #define sc_zend_hash_index_find zend_hash_index_find #define SC_MAKE_STD_ZVAL(p) zval _stack_zval_##p; p = &(_stack_zval_##p) diff --git a/typesToPhp.cpp b/typesToPhp.cpp index 25f5ac1..f6dcc90 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -636,7 +636,7 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) sc_add_assoc_stringl_ex(arr, column_name.c_str(), column_name.length(), val, len, 1); \ } -void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array, zend_long fetch_mode) +void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array, long fetch_mode) { switch (columnRef->Type()->GetCode()) { diff --git a/typesToPhp.hpp b/typesToPhp.hpp index 0a5f51b..c58307b 100644 --- a/typesToPhp.hpp +++ b/typesToPhp.hpp @@ -22,7 +22,7 @@ ColumnRef createColumn(TypeRef type); ColumnRef insertColumn(TypeRef type, zval *value_zval); -void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array, zend_long fetch_mode); +void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column_name, int8_t is_array, long fetch_mode); void zvalToBlock(Block& blockDes, Block& blockSrc, zend_ulong num_key, zval *value_zval); From 8c0a6da4cee274cbdb61637756607dea8a12a44a Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 15 Apr 2020 08:57:27 -0400 Subject: [PATCH 07/16] Permissions --- tests/012.phpt | 0 tests/013.phpt | 0 tests/014.phpt | 0 tests/015.phpt | 0 travis/run-tests.sh | 0 5 files changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 tests/012.phpt mode change 100755 => 100644 tests/013.phpt mode change 100755 => 100644 tests/014.phpt mode change 100755 => 100644 tests/015.phpt mode change 100755 => 100644 travis/run-tests.sh diff --git a/tests/012.phpt b/tests/012.phpt old mode 100755 new mode 100644 diff --git a/tests/013.phpt b/tests/013.phpt old mode 100755 new mode 100644 diff --git a/tests/014.phpt b/tests/014.phpt old mode 100755 new mode 100644 diff --git a/tests/015.phpt b/tests/015.phpt old mode 100755 new mode 100644 diff --git a/travis/run-tests.sh b/travis/run-tests.sh old mode 100755 new mode 100644 From a7a068fe59b7f665c7a92386123d34d03234b825 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 15 Apr 2020 13:28:55 -0400 Subject: [PATCH 08/16] Fixed typo in name Upgraded clickhouse lib to v1.1.0 Added ability control connection, recv, timeouts & retry settings --- lib/clickhouse-cpp/BUCK | 20 -- lib/clickhouse-cpp/CMakeLists.txt | 26 ++- lib/clickhouse-cpp/LICENSE | 2 +- lib/clickhouse-cpp/README.md | 9 +- lib/clickhouse-cpp/clickhouse/CMakeLists.txt | 13 +- lib/clickhouse-cpp/clickhouse/base/coded.cpp | 6 +- .../clickhouse/base/compressed.cpp | 8 - .../clickhouse/base/compressed.h | 1 - lib/clickhouse-cpp/clickhouse/base/platform.h | 2 +- lib/clickhouse-cpp/clickhouse/base/socket.cpp | 135 +++++++++---- lib/clickhouse-cpp/clickhouse/base/socket.h | 16 +- lib/clickhouse-cpp/clickhouse/client.cpp | 132 ++++++------- lib/clickhouse-cpp/clickhouse/client.h | 13 +- .../clickhouse/columns/array.cpp | 17 +- lib/clickhouse-cpp/clickhouse/columns/array.h | 8 +- .../clickhouse/columns/column.h | 6 +- .../clickhouse/columns/date.cpp | 3 +- .../clickhouse/columns/decimal.cpp | 98 ++++++++++ .../clickhouse/columns/decimal.h | 39 ++++ .../clickhouse/columns/factory.cpp | 29 ++- lib/clickhouse-cpp/clickhouse/columns/ip4.cpp | 81 ++++++++ lib/clickhouse-cpp/clickhouse/columns/ip4.h | 50 +++++ lib/clickhouse-cpp/clickhouse/columns/ip6.cpp | 81 ++++++++ lib/clickhouse-cpp/clickhouse/columns/ip6.h | 49 +++++ .../clickhouse/columns/nothing.h | 71 +++++++ .../clickhouse/columns/nullable.cpp | 1 + .../clickhouse/columns/numeric.cpp | 1 + .../clickhouse/columns/numeric.h | 5 + .../clickhouse/columns/string.cpp | 4 +- .../clickhouse/columns/string.h | 5 +- .../clickhouse/columns/tuple.cpp | 8 +- lib/clickhouse-cpp/clickhouse/columns/tuple.h | 7 +- .../clickhouse/columns/uuid.cpp | 6 +- lib/clickhouse-cpp/clickhouse/protocol.h | 92 +++++---- lib/clickhouse-cpp/clickhouse/query.cpp | 45 +++++ lib/clickhouse-cpp/clickhouse/query.h | 88 +++++---- .../clickhouse/types/type_parser.cpp | 12 +- .../clickhouse/types/type_parser.h | 4 +- lib/clickhouse-cpp/clickhouse/types/types.cpp | 42 ++++- lib/clickhouse-cpp/clickhouse/types/types.h | 26 +++ lib/clickhouse-cpp/cmake/cpp11.cmake | 8 - lib/clickhouse-cpp/cmake/cpp17.cmake | 8 + lib/clickhouse-cpp/contrib/cityhash/BUCK | 13 -- lib/clickhouse-cpp/contrib/gtest/BUCK | 14 -- lib/clickhouse-cpp/contrib/lz4/BUCK | 13 -- lib/clickhouse-cpp/tests/simple/BUCK | 12 -- lib/clickhouse-cpp/tests/simple/main.cpp | 177 +++++++++++++++++- lib/clickhouse-cpp/ut/BUCK | 12 -- lib/clickhouse-cpp/ut/CMakeLists.txt | 7 +- lib/clickhouse-cpp/ut/columns_ut.cpp | 10 +- lib/clickhouse-cpp/ut/socket_ut.cpp | 32 ++++ lib/clickhouse-cpp/ut/stream_ut.cpp | 23 +++ lib/clickhouse-cpp/ut/tcp_server.cpp | 56 ++++++ lib/clickhouse-cpp/ut/tcp_server.h | 21 +++ lib/clickhouse-cpp/ut/type_parser_ut.cpp | 60 ++++++ 55 files changed, 1363 insertions(+), 364 deletions(-) delete mode 100644 lib/clickhouse-cpp/BUCK create mode 100644 lib/clickhouse-cpp/clickhouse/columns/decimal.cpp create mode 100644 lib/clickhouse-cpp/clickhouse/columns/decimal.h create mode 100644 lib/clickhouse-cpp/clickhouse/columns/ip4.cpp create mode 100644 lib/clickhouse-cpp/clickhouse/columns/ip4.h create mode 100644 lib/clickhouse-cpp/clickhouse/columns/ip6.cpp create mode 100644 lib/clickhouse-cpp/clickhouse/columns/ip6.h create mode 100644 lib/clickhouse-cpp/clickhouse/columns/nothing.h delete mode 100644 lib/clickhouse-cpp/cmake/cpp11.cmake create mode 100644 lib/clickhouse-cpp/cmake/cpp17.cmake delete mode 100644 lib/clickhouse-cpp/contrib/cityhash/BUCK delete mode 100644 lib/clickhouse-cpp/contrib/gtest/BUCK delete mode 100644 lib/clickhouse-cpp/contrib/lz4/BUCK delete mode 100644 lib/clickhouse-cpp/tests/simple/BUCK delete mode 100644 lib/clickhouse-cpp/ut/BUCK create mode 100644 lib/clickhouse-cpp/ut/socket_ut.cpp create mode 100644 lib/clickhouse-cpp/ut/stream_ut.cpp create mode 100644 lib/clickhouse-cpp/ut/tcp_server.cpp create mode 100644 lib/clickhouse-cpp/ut/tcp_server.h diff --git a/lib/clickhouse-cpp/BUCK b/lib/clickhouse-cpp/BUCK deleted file mode 100644 index 0c86b12..0000000 --- a/lib/clickhouse-cpp/BUCK +++ /dev/null @@ -1,20 +0,0 @@ -cxx_library( - name = 'clickhouse-cpp', - header_namespace = 'clickhouse', - exported_headers = subdir_glob([ - ('clickhouse', '**/*.h'), - ]), - srcs = glob([ - 'clickhouse/**/*.cpp', - ]), - compiler_flags = [ - '-std=c++11', - ], - visibility = [ - 'PUBLIC', - ], - deps = [ - '//contrib/cityhash:cityhash', - '//contrib/lz4:lz4', - ] -) diff --git a/lib/clickhouse-cpp/CMakeLists.txt b/lib/clickhouse-cpp/CMakeLists.txt index fa3856f..8f967b0 100644 --- a/lib/clickhouse-cpp/CMakeLists.txt +++ b/lib/clickhouse-cpp/CMakeLists.txt @@ -1,21 +1,26 @@ CMAKE_MINIMUM_REQUIRED(VERSION 3.0.2) -INCLUDE (cmake/cpp11.cmake) +INCLUDE (cmake/cpp17.cmake) INCLUDE (cmake/subdirs.cmake) OPTION(BUILD_BENCHMARK "Build benchmark" OFF) +OPTION(BUILD_TESTS "Build tests" OFF) PROJECT (CLICKHOUSE-CLIENT) - USE_CXX11() + USE_CXX17() + + IF ("${CMAKE_BUILD_TYPE}" STREQUAL "") + set(CMAKE_BUILD_TYPE "Debug") + ENDIF() IF (UNIX) IF (APPLE) - SET (CMAKE_CXX_FLAGS "-O2 -Wall -Wextra -Werror") + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -Wall -Wextra -Werror") ELSE () - SET (CMAKE_CXX_FLAGS "-O2 -pthread -Wall -Wextra -Werror") + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O2 -pthread -Wall -Wextra -Werror") ENDIF () - SET (CMAKE_EXE_LINKER_FLAGS, "-lpthread") + SET (CMAKE_EXE_LINKER_FLAGS, "${CMAKE_EXE_LINKER_FLAGS} -lpthread") ENDIF () INCLUDE_DIRECTORIES(.) @@ -24,12 +29,17 @@ PROJECT (CLICKHOUSE-CLIENT) SUBDIRS ( clickhouse contrib/cityhash - contrib/gtest contrib/lz4 - tests/simple - ut ) IF (BUILD_BENCHMARK) SUBDIRS(bench) ENDIF (BUILD_BENCHMARK) + + IF (BUILD_TESTS) + SUBDIRS( + contrib/gtest + tests/simple + ut + ) + ENDIF (BUILD_TESTS) diff --git a/lib/clickhouse-cpp/LICENSE b/lib/clickhouse-cpp/LICENSE index 59dec86..1d426a7 100644 --- a/lib/clickhouse-cpp/LICENSE +++ b/lib/clickhouse-cpp/LICENSE @@ -1,4 +1,4 @@ -Copyright 2017 Pavel Artemkin +Copyright 2017 - 2020 Pavel Artemkin Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/lib/clickhouse-cpp/README.md b/lib/clickhouse-cpp/README.md index 1a840db..438ed5f 100644 --- a/lib/clickhouse-cpp/README.md +++ b/lib/clickhouse-cpp/README.md @@ -1,19 +1,18 @@ ClickHouse C++ client [![Build Status](https://travis-ci.org/artpaul/clickhouse-cpp.svg?branch=master)](https://travis-ci.org/artpaul/clickhouse-cpp) ===== -C++ client for [Yandex ClickHouse](https://clickhouse.yandex/) - -## This repositorie change -* [Add InsertQuery and InsertData methods](https://github.com/aiwhj/clickhouse-cpp/commit/bab28bcb5a509d80b8e2e0c7e89512446283dde5) +C++ client for [ClickHouse](https://clickhouse.tech/) ## Supported data types * Array(T) * Date * DateTime +* Decimal32, Decimal64, Decimal128 * Enum8, Enum16 * FixedString(N) * Float32, Float64 +* IPv4, IPv6 * Nullable(T) * String * Tuple @@ -24,7 +23,7 @@ C++ client for [Yandex ClickHouse](https://clickhouse.yandex/) ```sh $ mkdir build . $ cd build -$ cmake .. +$ cmake .. [-DBUILD_TESTS=ON] $ make ``` diff --git a/lib/clickhouse-cpp/clickhouse/CMakeLists.txt b/lib/clickhouse-cpp/clickhouse/CMakeLists.txt index d7a1de7..d5d1a00 100644 --- a/lib/clickhouse-cpp/clickhouse/CMakeLists.txt +++ b/lib/clickhouse-cpp/clickhouse/CMakeLists.txt @@ -1,4 +1,4 @@ -ADD_LIBRARY (clickhouse-cpp-lib +SET ( clickhouse-cpp-lib-src base/coded.cpp base/compressed.cpp base/input.cpp @@ -8,8 +8,11 @@ ADD_LIBRARY (clickhouse-cpp-lib columns/array.cpp columns/date.cpp + columns/decimal.cpp columns/enum.cpp columns/factory.cpp + columns/ip4.cpp + columns/ip6.cpp columns/nullable.cpp columns/numeric.cpp columns/string.cpp @@ -24,6 +27,8 @@ ADD_LIBRARY (clickhouse-cpp-lib query.cpp ) +ADD_LIBRARY (clickhouse-cpp-lib SHARED ${clickhouse-cpp-lib-src}) + SET_TARGET_PROPERTIES(clickhouse-cpp-lib PROPERTIES LINKER_LANGUAGE CXX) @@ -31,3 +36,9 @@ TARGET_LINK_LIBRARIES (clickhouse-cpp-lib cityhash-lib lz4-lib ) + +ADD_LIBRARY (clickhouse-cpp-lib-static STATIC ${clickhouse-cpp-lib-src}) +TARGET_LINK_LIBRARIES (clickhouse-cpp-lib-static + cityhash-lib + lz4-lib +) diff --git a/lib/clickhouse-cpp/clickhouse/base/coded.cpp b/lib/clickhouse-cpp/clickhouse/base/coded.cpp index 9e638ca..fbd29a2 100644 --- a/lib/clickhouse-cpp/clickhouse/base/coded.cpp +++ b/lib/clickhouse-cpp/clickhouse/base/coded.cpp @@ -45,13 +45,13 @@ bool CodedInputStream::Skip(size_t count) { bool CodedInputStream::ReadVarint64(uint64_t* value) { *value = 0; - for (size_t i = 0; i < 9; ++i) { + for (size_t i = 0; i < MAX_VARINT_BYTES; ++i) { uint8_t byte; if (!input_->ReadByte(&byte)) { return false; } else { - *value |= (byte & 0x7F) << (7 * i); + *value |= uint64_t(byte & 0x7F) << (7 * i); if (!(byte & 0x80)) { return true; @@ -81,7 +81,7 @@ void CodedOutputStream::WriteVarint64(uint64_t value) { uint8_t bytes[MAX_VARINT_BYTES]; int size = 0; - for (size_t i = 0; i < 9; ++i) { + for (size_t i = 0; i < MAX_VARINT_BYTES; ++i) { uint8_t byte = value & 0x7F; if (value > 0x7F) byte |= 0x80; diff --git a/lib/clickhouse-cpp/clickhouse/base/compressed.cpp b/lib/clickhouse-cpp/clickhouse/base/compressed.cpp index 33bc111..f1cb081 100644 --- a/lib/clickhouse-cpp/clickhouse/base/compressed.cpp +++ b/lib/clickhouse-cpp/clickhouse/base/compressed.cpp @@ -15,14 +15,6 @@ CompressedInput::CompressedInput(CodedInputStream* input) { } -CompressedInput::~CompressedInput() { - if (!mem_.Exhausted()) { - if (!std::uncaught_exception()) { - throw std::runtime_error("some data was not readed"); - } - } -} - size_t CompressedInput::DoNext(const void** ptr, size_t len) { if (mem_.Exhausted()) { if (!Decompress()) { diff --git a/lib/clickhouse-cpp/clickhouse/base/compressed.h b/lib/clickhouse-cpp/clickhouse/base/compressed.h index 8c1b461..ebbb9c1 100644 --- a/lib/clickhouse-cpp/clickhouse/base/compressed.h +++ b/lib/clickhouse-cpp/clickhouse/base/compressed.h @@ -7,7 +7,6 @@ namespace clickhouse { class CompressedInput : public ZeroCopyInput { public: CompressedInput(CodedInputStream* input); - ~CompressedInput(); protected: size_t DoNext(const void** ptr, size_t len) override; diff --git a/lib/clickhouse-cpp/clickhouse/base/platform.h b/lib/clickhouse-cpp/clickhouse/base/platform.h index e8bf4d1..9f53da1 100644 --- a/lib/clickhouse-cpp/clickhouse/base/platform.h +++ b/lib/clickhouse-cpp/clickhouse/base/platform.h @@ -15,7 +15,7 @@ # define _win_ #endif -#if defined(_linux_) +#if defined(_linux_) || defined (_darwin_) # define _unix_ #endif diff --git a/lib/clickhouse-cpp/clickhouse/base/socket.cpp b/lib/clickhouse-cpp/clickhouse/base/socket.cpp index 927f193..b8d4510 100644 --- a/lib/clickhouse-cpp/clickhouse/base/socket.cpp +++ b/lib/clickhouse-cpp/clickhouse/base/socket.cpp @@ -9,7 +9,9 @@ #if !defined(_win_) # include +# include # include +# include # include # include #endif @@ -17,23 +19,61 @@ namespace clickhouse { namespace { - class LocalNames : public std::unordered_set { - public: - LocalNames() { - emplace("localhost"); - emplace("localhost.localdomain"); - emplace("localhost6"); - emplace("localhost6.localdomain6"); - emplace("::1"); - emplace("127.0.0.1"); - } +class LocalNames : public std::unordered_set { +public: + LocalNames() { + emplace("localhost"); + emplace("localhost.localdomain"); + emplace("localhost6"); + emplace("localhost6.localdomain6"); + emplace("::1"); + emplace("127.0.0.1"); + } - inline bool IsLocalName(const std::string& name) const noexcept { - return find(name) != end(); + inline bool IsLocalName(const std::string& name) const noexcept { + return find(name) != end(); + } +}; + +void SetNonBlock(SOCKET fd, bool value) { +#if defined(_unix_) + int flags; + int ret; + #if defined(O_NONBLOCK) + if ((flags = fcntl(fd, F_GETFL, 0)) == -1) + flags = 0; + if (value) { + flags |= O_NONBLOCK; + } else { + flags &= ~O_NONBLOCK; } - }; + ret = fcntl(fd, F_SETFL, flags); + #else + flags = value; + return ioctl(fd, FIOBIO, &flags); + #endif + if (ret == -1) { + throw std::system_error( + errno, std::system_category(), "fail to set nonblocking mode"); + } +#elif defined(_win_) + unsigned long inbuf = value; + unsigned long outbuf = 0; + DWORD written = 0; + + if (!inbuf) { + WSAEventSelect(fd, nullptr, 0); + } + + if (WSAIoctl(fd, FIONBIO, &inbuf, sizeof(inbuf), &outbuf, sizeof(outbuf), &written, 0, 0) == SOCKET_ERROR) { + throw std::system_error( + errno, std::system_category(), "fail to set nonblocking mode"); + } +#endif } +} // namespace + NetworkAddress::NetworkAddress(const std::string& host, const std::string& port) : info_(nullptr) { @@ -83,7 +123,7 @@ SocketHolder::SocketHolder(SOCKET s) { } -SocketHolder::SocketHolder(SocketHolder&& other) +SocketHolder::SocketHolder(SocketHolder&& other) noexcept : handle_(other.handle_) { other.handle_ = -1; @@ -108,6 +148,26 @@ bool SocketHolder::Closed() const noexcept { return handle_ == -1; } +void SocketHolder::SetTcpKeepAlive(int idle, int intvl, int cnt) noexcept { + int val = 1; + +#if defined(_unix_) + setsockopt(handle_, SOL_SOCKET, SO_KEEPALIVE, &val, sizeof(val)); +# if defined(_linux_) + setsockopt(handle_, IPPROTO_TCP, TCP_KEEPIDLE, &idle, sizeof(idle)); +# elif defined(_darwin_) + setsockopt(handle_, IPPROTO_TCP, TCP_KEEPALIVE, &idle, sizeof(idle)); +# else +# error "platform does not supported" +# endif + setsockopt(handle_, IPPROTO_TCP, TCP_KEEPINTVL, &intvl, sizeof(intvl)); + setsockopt(handle_, IPPROTO_TCP, TCP_KEEPCNT, &cnt, sizeof(cnt)); +#else + setsockopt(handle_, SOL_SOCKET, SO_KEEPALIVE, (const char*)&val, sizeof(val)); + std::ignore = idle = intvl = cnt; +#endif +} + SocketHolder& SocketHolder::operator = (SocketHolder&& other) noexcept { if (this != &other) { Close(); @@ -164,7 +224,7 @@ void SocketOutput::DoWrite(const void* data, size_t len) { static const int flags = 0; #endif - if (::send(s_, (const char*)data, len, flags) != (int)len) { + if (::send(s_, (const char*)data, (int)len, flags) != (int)len) { throw std::system_error( errno, std::system_category(), "fail to send data" ); @@ -172,9 +232,9 @@ void SocketOutput::DoWrite(const void* data, size_t len) { } -NetrworkInitializer::NetrworkInitializer() { - struct NetrworkInitializerImpl { - NetrworkInitializerImpl() { +NetworkInitializer::NetworkInitializer() { + struct NetworkInitializerImpl { + NetworkInitializerImpl() { #if defined (_win_) WSADATA data; const int result = WSAStartup(MAKEWORD(2, 2), &data); @@ -189,11 +249,12 @@ NetrworkInitializer::NetrworkInitializer() { }; - (void)Singleton(); + (void)Singleton(); } SOCKET SocketConnect(const NetworkAddress& addr) { + int last_err = 0; for (auto res = addr.Info(); res != nullptr; res = res->ai_next) { SOCKET s(socket(res->ai_family, res->ai_socktype, res->ai_protocol)); @@ -201,31 +262,39 @@ SOCKET SocketConnect(const NetworkAddress& addr) { continue; } - if (connect(s, res->ai_addr, (int)res->ai_addrlen)) { - if (errno == EINPROGRESS || - errno == EAGAIN || - errno == EWOULDBLOCK) - { + SetNonBlock(s, true); + + if (connect(s, res->ai_addr, (int)res->ai_addrlen) != 0) { + int err = errno; + if (err == EINPROGRESS || err == EAGAIN || err == EWOULDBLOCK) { pollfd fd; fd.fd = s; fd.events = POLLOUT; - int rval = Poll(&fd, 1, 1000); + fd.revents = 0; + ssize_t rval = Poll(&fd, 1, 5000); + if (rval == -1) { + throw std::system_error(errno, std::system_category(), "fail to connect"); + } if (rval > 0) { - int opt; - socklen_t len = sizeof(opt); - getsockopt(s, SOL_SOCKET, SO_ERROR, (char*)&opt, &len); - - return opt; - } else { - continue; + socklen_t len = sizeof(err); + getsockopt(s, SOL_SOCKET, SO_ERROR, (char*)&err, &len); + + if (!err) { + SetNonBlock(s, false); + return s; + } + last_err = err; } } } else { + SetNonBlock(s, false); return s; } } - + if (last_err > 0) { + throw std::system_error(last_err, std::system_category(), "fail to connect"); + } throw std::system_error( errno, std::system_category(), "fail to connect" ); diff --git a/lib/clickhouse-cpp/clickhouse/base/socket.h b/lib/clickhouse-cpp/clickhouse/base/socket.h index f2fc32c..2d1bddd 100644 --- a/lib/clickhouse-cpp/clickhouse/base/socket.h +++ b/lib/clickhouse-cpp/clickhouse/base/socket.h @@ -13,6 +13,7 @@ # include # include #else +# include # include # include # include @@ -46,7 +47,7 @@ class SocketHolder { public: SocketHolder(); SocketHolder(SOCKET s); - SocketHolder(SocketHolder&& other); + SocketHolder(SocketHolder&& other) noexcept; ~SocketHolder(); @@ -54,6 +55,13 @@ class SocketHolder { bool Closed() const noexcept; + /// @params idle the time (in seconds) the connection needs to remain + /// idle before TCP starts sending keepalive probes. + /// @params intvl the time (in seconds) between individual keepalive probes. + /// @params cnt the maximum number of keepalive probes TCP should send + /// before dropping the connection. + void SetTcpKeepAlive(int idle, int intvl, int cnt) noexcept; + SocketHolder& operator = (SocketHolder&& other) noexcept; operator SOCKET () const noexcept; @@ -93,9 +101,9 @@ class SocketOutput : public OutputStream { SOCKET s_; }; -static struct NetrworkInitializer { - NetrworkInitializer(); -} gNetrworkInitializer; +static struct NetworkInitializer { + NetworkInitializer(); +} gNetworkInitializer; /// SOCKET SocketConnect(const NetworkAddress& addr); diff --git a/lib/clickhouse-cpp/clickhouse/client.cpp b/lib/clickhouse-cpp/clickhouse/client.cpp index 03111b5..f2c62d1 100644 --- a/lib/clickhouse-cpp/clickhouse/client.cpp +++ b/lib/clickhouse-cpp/clickhouse/client.cpp @@ -17,7 +17,6 @@ #include #include #include -#include #define DBMS_NAME "ClickHouse" #define DBMS_VERSION_MAJOR 1 @@ -25,7 +24,6 @@ #define REVISION 54126 #define DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES 50264 -#define DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS 51554 #define DBMS_MIN_REVISION_WITH_BLOCK_INFO 51903 #define DBMS_MIN_REVISION_WITH_CLIENT_INFO 54032 #define DBMS_MIN_REVISION_WITH_SERVER_TIMEZONE 54058 @@ -78,10 +76,6 @@ class Client::Impl { void Insert(const std::string& table_name, const Block& block); - void InsertQuery(Query query); - - void InsertData(const Block& block); - void Ping(); void ResetConnection(); @@ -102,7 +96,7 @@ class Client::Impl { bool ReceiveHello(); /// Reads data packet form input stream. - bool ReceiveData(); + bool ReceiveData(std::function cb); /// Reads exception packet form input stream. bool ReceiveException(bool rethrow = false); @@ -110,10 +104,6 @@ class Client::Impl { void WriteBlock(const Block& block, CodedOutputStream* output); private: - void Disconnect() { - socket_.Close(); - } - /// In case of network errors tries to reconnect to server and /// call fuc several times. void RetryGuard(std::function fuc); @@ -188,9 +178,8 @@ Client::Impl::Impl(const ClientOptions& opts) } } -Client::Impl::~Impl() { - Disconnect(); -} +Client::Impl::~Impl() +{ } void Client::Impl::ExecuteQuery(Query query) { EnsureNull en(static_cast(&query), &events_); @@ -259,45 +248,6 @@ void Client::Impl::Insert(const std::string& table_name, const Block& block) { } } -void Client::Impl::InsertQuery(Query query) { - EnsureNull en(static_cast(&query), &events_); - - if (options_.ping_before_query) { - RetryGuard([this]() { Ping(); }); - } - - SendQuery(query.GetText()); - - uint64_t server_packet; - // Receive data packet. - while (true) { - bool ret = ReceivePacket(&server_packet); - - if (!ret) { - throw std::runtime_error("fail to receive data packet"); - } - if (server_packet == ServerCodes::Data) { - break; - } - if (server_packet == ServerCodes::Progress) { - continue; - } - } -} - -void Client::Impl::InsertData(const Block& block) { - // Send data. - SendData(block); - // Send empty block as marker of - // end of data. - SendData(Block()); - - // Wait for EOS. - while (ReceivePacket()) { - ; - } -} - void Client::Impl::Ping() { WireFormat::WriteUInt64(&output_, ClientCodes::Ping); output_.Flush(); @@ -317,6 +267,12 @@ void Client::Impl::ResetConnection() { throw std::system_error(errno, std::system_category()); } + if (options_.tcp_keepalive) { + s.SetTcpKeepAlive(options_.tcp_keepalive_idle.count(), + options_.tcp_keepalive_intvl.count(), + options_.tcp_keepalive_cnt); + } + socket_ = std::move(s); socket_input_ = SocketInput(socket_); socket_output_ = SocketOutput(socket_); @@ -350,7 +306,16 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { switch (packet_type) { case ServerCodes::Data: { - if (!ReceiveData()) { + auto cb = [this] (const Block& block) { + if (events_) { + events_->OnData(block); + if (!events_->OnDataCancelable(block)) { + SendCancel(); + } + } + }; + + if (!ReceiveData(cb)) { throw std::runtime_error("can't read data packet from input stream"); } return true; @@ -399,10 +364,8 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { if (!WireFormat::ReadUInt64(&input_, &info.bytes)) { return false; } - if (REVISION >= DBMS_MIN_REVISION_WITH_TOTAL_ROWS_IN_PROGRESS) { - if (!WireFormat::ReadUInt64(&input_, &info.total_rows)) { - return false; - } + if (!WireFormat::ReadUInt64(&input_, &info.total_rows)) { + return false; } if (events_) { @@ -423,6 +386,32 @@ bool Client::Impl::ReceivePacket(uint64_t* server_packet) { return false; } + case ServerCodes::Totals: { + auto cb = [this] (const Block& block) { + if (events_) { + events_->OnTotals(block); + } + }; + + if (!ReceiveData(cb)) { + throw std::runtime_error("can't read data packet with totals from input stream"); + } + return true; + } + + case ServerCodes::Extremes: { + auto cb = [this] (const Block& block) { + if (events_) { + events_->OnExtremes(block); + } + }; + + if (!ReceiveData(cb)) { + throw std::runtime_error("can't read data packet with extremes from input stream"); + } + return true; + } + default: throw std::runtime_error("unimplemented " + std::to_string((int)packet_type)); break; @@ -492,15 +481,13 @@ bool Client::Impl::ReadBlock(Block* block, CodedInputStream* input) { return true; } -bool Client::Impl::ReceiveData() { +bool Client::Impl::ReceiveData(std::function cb) { Block block; + std::string table_name; - if (REVISION >= DBMS_MIN_REVISION_WITH_TEMPORARY_TABLES) { - std::string table_name; - - if (!WireFormat::ReadString(&input_, &table_name)) { - return false; - } + // Read name of a table. + if (!WireFormat::ReadString(&input_, &table_name)) { + return false; } if (compression_ == CompressionState::Enable) { @@ -516,12 +503,7 @@ bool Client::Impl::ReceiveData() { } } - if (events_) { - events_->OnData(block); - if (!events_->OnDataCancelable(block)) { - SendCancel(); - } - } + cb(block); return true; } @@ -797,14 +779,6 @@ void Client::Insert(const std::string& table_name, const Block& block) { impl_->Insert(table_name, block); } -void Client::InsertQuery(const std::string& query, SelectCallback cb) { - impl_->InsertQuery(Query(query).OnData(cb)); -} - -void Client::InsertData(const Block& block) { - impl_->InsertData(block); -} - void Client::Ping() { impl_->Ping(); } diff --git a/lib/clickhouse-cpp/clickhouse/client.h b/lib/clickhouse-cpp/clickhouse/client.h index 2f4352d..6953e7d 100644 --- a/lib/clickhouse-cpp/clickhouse/client.h +++ b/lib/clickhouse-cpp/clickhouse/client.h @@ -5,7 +5,10 @@ #include "columns/array.h" #include "columns/date.h" +#include "columns/decimal.h" #include "columns/enum.h" +#include "columns/ip4.h" +#include "columns/ip6.h" #include "columns/nullable.h" #include "columns/numeric.h" #include "columns/string.h" @@ -60,6 +63,12 @@ struct ClientOptions { /// Compression method. DECLARE_FIELD(compression_method, CompressionMethod, SetCompressionMethod, CompressionMethod::None); + /// TCP Keep alive options + DECLARE_FIELD(tcp_keepalive, bool, TcpKeepAlive, false); + DECLARE_FIELD(tcp_keepalive_idle, std::chrono::seconds, SetTcpKeepAliveIdle, std::chrono::seconds(60)); + DECLARE_FIELD(tcp_keepalive_intvl, std::chrono::seconds, SetTcpKeepAliveInterval, std::chrono::seconds(5)); + DECLARE_FIELD(tcp_keepalive_cnt, int, SetTcpKeepAliveCount, 3); + #undef DECLARE_FIELD }; @@ -89,10 +98,6 @@ class Client { /// Intends for insert block of data into a table \p table_name. void Insert(const std::string& table_name, const Block& block); - - void InsertQuery(const std::string& query, SelectCallback cb); - - void InsertData(const Block& block); /// Ping server for aliveness. void Ping(); diff --git a/lib/clickhouse-cpp/clickhouse/columns/array.cpp b/lib/clickhouse-cpp/clickhouse/columns/array.cpp index ab7d145..7af64dd 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/array.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/array.cpp @@ -1,5 +1,4 @@ #include "array.h" - #include namespace clickhouse { @@ -31,6 +30,18 @@ ColumnRef ColumnArray::GetAsColumn(size_t n) const { return data_->Slice(GetOffset(n), GetSize(n)); } +ColumnRef ColumnArray::Slice(size_t begin, size_t size) { + auto result = std::make_shared(GetAsColumn(begin)); + result->OffsetsIncrease(1); + + for (size_t i = 1; i < size; i++) + { + result->Append(std::make_shared(GetAsColumn(begin + i))); + } + + return result; +} + void ColumnArray::Append(ColumnRef column) { if (auto col = column->As()) { if (!col->data_->Type()->IsEqual(data_->Type())) { @@ -67,6 +78,10 @@ size_t ColumnArray::Size() const { return offsets_->Size(); } +void ColumnArray::OffsetsIncrease(size_t n) { + offsets_->Append(n); +} + size_t ColumnArray::GetOffset(size_t n) const { return (n == 0) ? 0 : (*offsets_)[n - 1]; } diff --git a/lib/clickhouse-cpp/clickhouse/columns/array.h b/lib/clickhouse-cpp/clickhouse/columns/array.h index c631bb2..50ddcab 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/array.h +++ b/lib/clickhouse-cpp/clickhouse/columns/array.h @@ -28,15 +28,17 @@ class ColumnArray : public Column { /// Saves column data to output stream. void Save(CodedOutputStream* output) override; - + /// Clear column data . void Clear() override; - + /// Returns count of rows in the column. size_t Size() const override; /// Makes slice of the current column. - ColumnRef Slice(size_t, size_t) override { return ColumnRef(); } + ColumnRef Slice(size_t, size_t) override; + + void OffsetsIncrease(size_t); private: size_t GetOffset(size_t n) const; diff --git a/lib/clickhouse-cpp/clickhouse/columns/column.h b/lib/clickhouse-cpp/clickhouse/columns/column.h index efec2eb..7ba1247 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/column.h +++ b/lib/clickhouse-cpp/clickhouse/columns/column.h @@ -22,13 +22,13 @@ class Column : public std::enable_shared_from_this virtual ~Column() { } - /// Downcast pointer to the specific culumn's subtype. + /// Downcast pointer to the specific column's subtype. template inline std::shared_ptr As() { return std::dynamic_pointer_cast(shared_from_this()); } - /// Downcast pointer to the specific culumn's subtype. + /// Downcast pointer to the specific column's subtype. template inline std::shared_ptr As() const { return std::dynamic_pointer_cast(shared_from_this()); @@ -45,7 +45,7 @@ class Column : public std::enable_shared_from_this /// Saves column data to output stream. virtual void Save(CodedOutputStream* output) = 0; - + /// Clear column data . virtual void Clear() = 0; diff --git a/lib/clickhouse-cpp/clickhouse/columns/date.cpp b/lib/clickhouse-cpp/clickhouse/columns/date.cpp index eab41c8..7bf30c4 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/date.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/date.cpp @@ -17,7 +17,7 @@ void ColumnDate::Clear() { } std::time_t ColumnDate::At(size_t n) const { - return data_->At(n) * 86400; + return static_cast(data_->At(n)) * 86400; } void ColumnDate::Append(ColumnRef column) { @@ -93,5 +93,4 @@ ColumnRef ColumnDateTime::Slice(size_t begin, size_t len) { return result; } - } diff --git a/lib/clickhouse-cpp/clickhouse/columns/decimal.cpp b/lib/clickhouse-cpp/clickhouse/columns/decimal.cpp new file mode 100644 index 0000000..e13d815 --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/decimal.cpp @@ -0,0 +1,98 @@ +#include "decimal.h" + +namespace clickhouse { + +ColumnDecimal::ColumnDecimal(size_t precision, size_t scale) + : Column(Type::CreateDecimal(precision, scale)) +{ + if (precision <= 9) { + data_ = std::make_shared(); + } else if (precision <= 18) { + data_ = std::make_shared(); + } else { + data_ = std::make_shared(); + } +} + +ColumnDecimal::ColumnDecimal(TypeRef type) + : Column(type) +{ +} + +void ColumnDecimal::Append(const Int128& value) { + if (data_->Type()->GetCode() == Type::Int32) { + data_->As()->Append(static_cast(value)); + } else if (data_->Type()->GetCode() == Type::Int64) { + data_->As()->Append(static_cast(value)); + } else { + data_->As()->Append(static_cast(value)); + } +} + +void ColumnDecimal::Append(const std::string& value) { + Int128 int_value = 0; + auto c = value.begin(); + bool sign = true; + + while (c != value.end()) { + if (*c == '-') { + sign = false; + if (c != value.begin()) { + break; + } + } else if (*c == '.') { + // TODO: compare distance with `scale` + } else if (*c >= '0' && *c <= '9') { + int_value = int_value * 10 + (*c - '0'); + } else { + // TODO: throw exception on unexpected symbol + } + ++c; + } + + if (c != value.end()) { + // TODO: throw exception about symbols after 'minus' + } + + Append(sign ? int_value : -int_value); +} + +Int128 ColumnDecimal::At(size_t i) const { + if (data_->Type()->GetCode() == Type::Int32) { + return static_cast(data_->As()->At(i)); + } else if (data_->Type()->GetCode() == Type::Int64) { + return static_cast(data_->As()->At(i)); + } else { + return data_->As()->At(i); + } +} + +void ColumnDecimal::Append(ColumnRef column) { + if (auto col = column->As()) { + data_->Append(col->data_); + } +} + +bool ColumnDecimal::Load(CodedInputStream* input, size_t rows) { + return data_->Load(input, rows); +} + +void ColumnDecimal::Save(CodedOutputStream* output) { + data_->Save(output); +} + +void ColumnDecimal::Clear() { + data_->Clear(); +} + +size_t ColumnDecimal::Size() const { + return data_->Size(); +} + +ColumnRef ColumnDecimal::Slice(size_t begin, size_t len) { + std::shared_ptr slice(new ColumnDecimal(type_)); + slice->data_ = data_->Slice(begin, len); + return slice; +} + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/decimal.h b/lib/clickhouse-cpp/clickhouse/columns/decimal.h new file mode 100644 index 0000000..9137a44 --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/decimal.h @@ -0,0 +1,39 @@ + +#pragma once + +#include "column.h" +#include "numeric.h" + +namespace clickhouse { + +/** + * Represents a column of decimal type. + */ +class ColumnDecimal : public Column { +public: + ColumnDecimal(size_t precision, size_t scale); + + void Append(const Int128& value); + void Append(const std::string& value); + + Int128 At(size_t i) const; + +public: + void Append(ColumnRef column) override; + bool Load(CodedInputStream* input, size_t rows) override; + void Save(CodedOutputStream* output) override; + void Clear() override; + size_t Size() const override; + ColumnRef Slice(size_t begin, size_t len) override; + +private: + /// Depending on a precision it can be one of: + /// - ColumnInt32 + /// - ColumnInt64 + /// - ColumnInt128 + ColumnRef data_; + + explicit ColumnDecimal(TypeRef type); // for `Slice(…)` +}; + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/factory.cpp b/lib/clickhouse-cpp/clickhouse/columns/factory.cpp index 47dddf9..055c8eb 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/factory.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/factory.cpp @@ -2,7 +2,11 @@ #include "array.h" #include "date.h" +#include "decimal.h" #include "enum.h" +#include "ip4.h" +#include "ip6.h" +#include "nothing.h" #include "nullable.h" #include "numeric.h" #include "string.h" @@ -16,6 +20,9 @@ namespace { static ColumnRef CreateTerminalColumn(const TypeAst& ast) { switch (ast.code) { + case Type::Void: + return std::make_shared(); + case Type::UInt8: return std::make_shared(); case Type::UInt16: @@ -34,14 +41,20 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::Int64: return std::make_shared(); - case Type::UUID: - return std::make_shared(); - case Type::Float32: return std::make_shared(); case Type::Float64: return std::make_shared(); + case Type::Decimal: + return std::make_shared(ast.elements.front().value, ast.elements.back().value); + case Type::Decimal32: + return std::make_shared(9, ast.elements.front().value); + case Type::Decimal64: + return std::make_shared(18, ast.elements.front().value); + case Type::Decimal128: + return std::make_shared(38, ast.elements.front().value); + case Type::String: return std::make_shared(); case Type::FixedString: @@ -52,6 +65,14 @@ static ColumnRef CreateTerminalColumn(const TypeAst& ast) { case Type::Date: return std::make_shared(); + case Type::IPv4: + return std::make_shared(); + case Type::IPv6: + return std::make_shared(); + + case Type::UUID: + return std::make_shared(); + default: return nullptr; } @@ -79,6 +100,7 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast) { case TypeAst::Tuple: { std::vector columns; + columns.reserve(ast.elements.size()); for (const auto& elem : ast.elements) { if (auto col = CreateColumnFromAst(elem)) { columns.push_back(col); @@ -93,6 +115,7 @@ static ColumnRef CreateColumnFromAst(const TypeAst& ast) { case TypeAst::Enum: { std::vector enum_items; + enum_items.reserve(ast.elements.size()); for (const auto& elem : ast.elements) { enum_items.push_back( Type::EnumItem{elem.name, (int16_t)elem.value}); diff --git a/lib/clickhouse-cpp/clickhouse/columns/ip4.cpp b/lib/clickhouse-cpp/clickhouse/columns/ip4.cpp new file mode 100644 index 0000000..c39c921 --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/ip4.cpp @@ -0,0 +1,81 @@ + +#include "ip4.h" + +#include + +#if defined(_win_) +using in_addr_t = unsigned long; +#endif + +namespace clickhouse { + +ColumnIPv4::ColumnIPv4() + : Column(Type::CreateIPv4()) + , data_(std::make_shared()) +{ +} + +ColumnIPv4::ColumnIPv4(ColumnRef data) + : Column(Type::CreateIPv4()) + , data_(data->As()) +{ + if (data_->Size() != 0) { + throw std::runtime_error("number of entries must be even (32-bit numbers for each IPv4)"); + } +} + +void ColumnIPv4::Append(const std::string& str) { + in_addr_t addr = inet_addr(str.c_str()); + if (addr == INADDR_NONE) { + throw std::runtime_error("invalid IPv4 format, ip: " + str); + } + data_->Append(htonl(addr)); +} + +void ColumnIPv4::Append(uint32_t ip) { + data_->Append(htonl(ip)); +} + +void ColumnIPv4::Clear() { + data_->Clear(); +} + +in_addr ColumnIPv4::At(size_t n) const { + struct in_addr addr; + addr.s_addr = ntohl(data_->At(n)); + return addr; +} + +in_addr ColumnIPv4::operator [] (size_t n) const { + struct in_addr addr; + addr.s_addr = ntohl(data_->operator[](n)); + return addr; +} + +std::string ColumnIPv4::AsString(size_t n) const { + return inet_ntoa(this->At(n)); +} + +void ColumnIPv4::Append(ColumnRef column) { + if (auto col = column->As()) { + data_->Append(col->data_); + } +} + +bool ColumnIPv4::Load(CodedInputStream* input, size_t rows) { + return data_->Load(input, rows); +} + +void ColumnIPv4::Save(CodedOutputStream* output) { + data_->Save(output); +} + +size_t ColumnIPv4::Size() const { + return data_->Size(); +} + +ColumnRef ColumnIPv4::Slice(size_t begin, size_t len) { + return std::make_shared(data_->Slice(begin, len)); +} + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/ip4.h b/lib/clickhouse-cpp/clickhouse/columns/ip4.h new file mode 100644 index 0000000..b4acae5 --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/ip4.h @@ -0,0 +1,50 @@ +#pragma once + +#include "numeric.h" +#include "../base/socket.h" + +namespace clickhouse { + +class ColumnIPv4 : public Column { +public: + ColumnIPv4(); + explicit ColumnIPv4(ColumnRef data); + + /// Appends one element to the column. + void Append(const std::string& ip); + + /// @params ip numeric value with host byte order. + void Append(uint32_t ip); + + /// Returns element at given row number. + in_addr At(size_t n) const; + + /// Returns element at given row number. + in_addr operator [] (size_t n) const; + + std::string AsString(size_t n) const; + +public: + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column data from input stream. + bool Load(CodedInputStream* input, size_t rows) override; + + /// Saves column data to output stream. + void Save(CodedOutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t begin, size_t len) override; + +private: + std::shared_ptr data_; +}; + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/ip6.cpp b/lib/clickhouse-cpp/clickhouse/columns/ip6.cpp new file mode 100644 index 0000000..47e0744 --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/ip6.cpp @@ -0,0 +1,81 @@ + +#include "ip6.h" + +#include + +namespace clickhouse { + +static_assert(sizeof(struct in6_addr) == 16, "sizeof in6_addr should be 16 bytes"); + +ColumnIPv6::ColumnIPv6() + : Column(Type::CreateIPv6()) + , data_(std::make_shared(16)) +{ +} + +ColumnIPv6::ColumnIPv6(ColumnRef data) + : Column(Type::CreateIPv6()) + , data_(data->As()) +{ + if (data_->Size() != 0) { + throw std::runtime_error("number of entries must be even (two 64-bit numbers for each IPv6)"); + } +} + +void ColumnIPv6::Append(const std::string& ip) { + unsigned char buf[16]; + if (inet_pton(AF_INET6, ip.c_str(), buf) != 1) { + throw std::runtime_error("invalid IPv6 format, ip: " + ip); + } + data_->Append(std::string((const char*)buf, 16)); +} + +void ColumnIPv6::Append(const in6_addr* addr) { + data_->Append(std::string((const char*)addr->s6_addr, 16)); +} + +void ColumnIPv6::Clear() { + data_->Clear(); +} + +std::string ColumnIPv6::AsString (size_t n) const{ + const auto& addr = data_->At(n); + char buf[INET6_ADDRSTRLEN]; + const char* ip_str = inet_ntop(AF_INET6, addr.data(), buf, INET6_ADDRSTRLEN); + if (ip_str == nullptr) { + throw std::runtime_error("invalid IPv6 format: " + addr); + } + return ip_str; +} + +in6_addr ColumnIPv6::At(size_t n) const { + return *reinterpret_cast(data_->At(n).data()); +} + +in6_addr ColumnIPv6::operator [] (size_t n) const { + return *reinterpret_cast(data_->At(n).data()); +} + +void ColumnIPv6::Append(ColumnRef column) { + if (auto col = column->As()) { + data_->Append(col->data_); + } +} + +bool ColumnIPv6::Load(CodedInputStream* input, size_t rows) { + return data_->Load(input, rows); +} + +void ColumnIPv6::Save(CodedOutputStream* output) { + data_->Save(output); +} + +size_t ColumnIPv6::Size() const { + return data_->Size(); +} + +ColumnRef ColumnIPv6::Slice(size_t begin, size_t len) { + return std::make_shared(data_->Slice(begin, len)); +} + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/ip6.h b/lib/clickhouse-cpp/clickhouse/columns/ip6.h new file mode 100644 index 0000000..0686c7e --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/ip6.h @@ -0,0 +1,49 @@ +#pragma once + +#include "string.h" +#include "../base/socket.h" + +namespace clickhouse { + +class ColumnIPv6 : public Column{ +public: + ColumnIPv6(); + explicit ColumnIPv6(ColumnRef data); + + /// Appends one element to the column. + void Append(const std::string& str); + + void Append(const in6_addr* addr); + + /// Returns element at given row number. + in6_addr At(size_t n) const; + + /// Returns element at given row number. + in6_addr operator [] (size_t n) const; + + std::string AsString(size_t n) const; + +public: + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override; + + /// Loads column data from input stream. + bool Load(CodedInputStream* input, size_t rows) override; + + /// Saves column data to output stream. + void Save(CodedOutputStream* output) override; + + /// Clear column data . + void Clear() override; + + /// Returns count of rows in the column. + size_t Size() const override; + + /// Makes slice of the current column. + ColumnRef Slice(size_t begin, size_t len) override; + +private: + std::shared_ptr data_; +}; + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/nothing.h b/lib/clickhouse-cpp/clickhouse/columns/nothing.h new file mode 100644 index 0000000..a08fc0d --- /dev/null +++ b/lib/clickhouse-cpp/clickhouse/columns/nothing.h @@ -0,0 +1,71 @@ + +#pragma once + +#include "column.h" + +#include + +namespace clickhouse { + +/** + * Represents dummy column of NULLs. + */ +class ColumnNothing : public Column { +public: + ColumnNothing() + : Column(Type::CreateNothing()) + , size_(0) + { + } + + explicit ColumnNothing(size_t n) + : Column(Type::CreateNothing()) + , size_(n) + { + } + + /// Appends one element to the column. + void Append(std::unique_ptr) { ++size_; } + + /// Returns element at given row number. + std::nullptr_t At(size_t) const { return nullptr; }; + + /// Returns element at given row number. + std::nullptr_t operator [] (size_t) const { return nullptr; }; + + /// Makes slice of the current column. + ColumnRef Slice(size_t, size_t len) override { + return std::make_shared(len); + } + +public: + /// Appends content of given column to the end of current one. + void Append(ColumnRef column) override { + if (auto col = column->As()) { + size_ += col->Size(); + } + } + + /// Loads column data from input stream. + bool Load(CodedInputStream* input, size_t rows) override { + input->Skip(rows); + size_ += rows; + return true; + } + + /// Saves column data to output stream. + void Save(CodedOutputStream*) override { + throw std::runtime_error("method Save is not supported for Nothing column"); + } + + /// Clear column data . + void Clear() override { size_ = 0; } + + /// Returns count of rows in the column. + size_t Size() const override { return size_; } + +private: + size_t size_; +}; + +} diff --git a/lib/clickhouse-cpp/clickhouse/columns/nullable.cpp b/lib/clickhouse-cpp/clickhouse/columns/nullable.cpp index 5f058d4..bec1a18 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/nullable.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/nullable.cpp @@ -1,6 +1,7 @@ #include "nullable.h" #include +#include namespace clickhouse { diff --git a/lib/clickhouse-cpp/clickhouse/columns/numeric.cpp b/lib/clickhouse-cpp/clickhouse/columns/numeric.cpp index bbd516d..52ecf7b 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/numeric.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/numeric.cpp @@ -74,6 +74,7 @@ template class ColumnVector; template class ColumnVector; template class ColumnVector; template class ColumnVector; +template class ColumnVector; template class ColumnVector; template class ColumnVector; diff --git a/lib/clickhouse-cpp/clickhouse/columns/numeric.h b/lib/clickhouse-cpp/clickhouse/columns/numeric.h index 45e3e30..ab9a11f 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/numeric.h +++ b/lib/clickhouse-cpp/clickhouse/columns/numeric.h @@ -10,6 +10,8 @@ namespace clickhouse { template class ColumnVector : public Column { public: + using DataType = T; + ColumnVector(); explicit ColumnVector(const std::vector& data); @@ -46,6 +48,8 @@ class ColumnVector : public Column { std::vector data_; }; +using Int128 = __int128; + using ColumnUInt8 = ColumnVector; using ColumnUInt16 = ColumnVector; using ColumnUInt32 = ColumnVector; @@ -55,6 +59,7 @@ using ColumnInt8 = ColumnVector; using ColumnInt16 = ColumnVector; using ColumnInt32 = ColumnVector; using ColumnInt64 = ColumnVector; +using ColumnInt128 = ColumnVector; using ColumnFloat32 = ColumnVector; using ColumnFloat64 = ColumnVector; diff --git a/lib/clickhouse-cpp/clickhouse/columns/string.cpp b/lib/clickhouse-cpp/clickhouse/columns/string.cpp index 0791224..3ad45a0 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/string.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/string.cpp @@ -45,7 +45,7 @@ bool ColumnFixedString::Load(CodedInputStream* input, size_t rows) { return false; } - data_.push_back(s); + data_.push_back(std::move(s)); } return true; @@ -113,7 +113,7 @@ bool ColumnString::Load(CodedInputStream* input, size_t rows) { return false; } - data_.push_back(s); + data_.push_back(std::move(s)); } return true; diff --git a/lib/clickhouse-cpp/clickhouse/columns/string.h b/lib/clickhouse-cpp/clickhouse/columns/string.h index 196ecba..8dcb74b 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/string.h +++ b/lib/clickhouse-cpp/clickhouse/columns/string.h @@ -29,7 +29,7 @@ class ColumnFixedString : public Column { /// Saves column data to output stream. void Save(CodedOutputStream* output) override; - + /// Clear column data . void Clear() override; @@ -44,7 +44,6 @@ class ColumnFixedString : public Column { std::vector data_; }; - /** * Represents column of variable-length strings. */ @@ -71,7 +70,7 @@ class ColumnString : public Column { /// Saves column data to output stream. void Save(CodedOutputStream* output) override; - + /// Clear column data . void Clear() override; diff --git a/lib/clickhouse-cpp/clickhouse/columns/tuple.cpp b/lib/clickhouse-cpp/clickhouse/columns/tuple.cpp index 9a009e3..5d467c0 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/tuple.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/tuple.cpp @@ -16,12 +16,12 @@ ColumnTuple::ColumnTuple(const std::vector& columns) { } -size_t ColumnTuple::Size() const { - return columns_.empty() ? 0 : columns_[0]->Size(); +size_t ColumnTuple::TupleSize() const { + return columns_.size(); } -size_t ColumnTuple::tupleSize() { - return columns_.empty() ? 0 : columns_.size(); +size_t ColumnTuple::Size() const { + return columns_.empty() ? 0 : columns_[0]->Size(); } bool ColumnTuple::Load(CodedInputStream* input, size_t rows) { diff --git a/lib/clickhouse-cpp/clickhouse/columns/tuple.h b/lib/clickhouse-cpp/clickhouse/columns/tuple.h index f261c52..b2bbad5 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/tuple.h +++ b/lib/clickhouse-cpp/clickhouse/columns/tuple.h @@ -13,12 +13,13 @@ class ColumnTuple : public Column { public: ColumnTuple(const std::vector& columns); + /// Returns count of columns in the tuple. + size_t TupleSize() const; + ColumnRef operator [] (size_t n) const { return columns_[n]; } - size_t tupleSize(); - public: /// Appends content of given column to the end of current one. void Append(ColumnRef) override { } @@ -28,7 +29,7 @@ class ColumnTuple : public Column { /// Saves column data to output stream. void Save(CodedOutputStream* output) override; - + /// Clear column data . void Clear() override; diff --git a/lib/clickhouse-cpp/clickhouse/columns/uuid.cpp b/lib/clickhouse-cpp/clickhouse/columns/uuid.cpp index b47592a..0c19781 100644 --- a/lib/clickhouse-cpp/clickhouse/columns/uuid.cpp +++ b/lib/clickhouse-cpp/clickhouse/columns/uuid.cpp @@ -1,6 +1,8 @@ #include "uuid.h" #include "utils.h" +#include + namespace clickhouse { ColumnUUID::ColumnUUID() @@ -13,7 +15,7 @@ ColumnUUID::ColumnUUID(ColumnRef data) : Column(Type::CreateUUID()) , data_(data->As()) { - if (data_->Size()%2 != 0) { + if (data_->Size() % 2 != 0) { throw std::runtime_error("number of entries must be even (two 64-bit numbers for each UUID)"); } } @@ -37,7 +39,7 @@ const UInt128 ColumnUUID::operator [] (size_t n) const { void ColumnUUID::Append(ColumnRef column) { if (auto col = column->As()) { - data_->Append(data_); + data_->Append(col->data_); } } diff --git a/lib/clickhouse-cpp/clickhouse/protocol.h b/lib/clickhouse-cpp/clickhouse/protocol.h index 46ae7ce..d5f9c26 100644 --- a/lib/clickhouse-cpp/clickhouse/protocol.h +++ b/lib/clickhouse-cpp/clickhouse/protocol.h @@ -2,46 +2,60 @@ namespace clickhouse { - /// То, что передаёт сервер. - namespace ServerCodes { - enum { - Hello = 0, /// Имя, версия, ревизия. - Data = 1, /// Блок данных со сжатием или без. - Exception = 2, /// Исключение во время обработки запроса. - Progress = 3, /// Прогресс выполнения запроса: строк считано, байт считано. - Pong = 4, /// Ответ на Ping. - EndOfStream = 5, /// Все пакеты были переданы. - ProfileInfo = 6, /// Пакет с профайлинговой информацией. - Totals = 7, /// Блок данных с тотальными значениями, со сжатием или без. - Extremes = 8, /// Блок данных с минимумами и максимумами, аналогично. - }; - } +/// Packet types that server transmits. +namespace ServerCodes { + enum { + /// Name, version, revision. + Hello = 0, + /// A block of data (compressed or not). + Data = 1, + /// An exception during query execution. + Exception = 2, + /// Query execution progress: rows read, bytes read. + Progress = 3, + /// Ping response. + Pong = 4, + /// All packets were transmitted. + EndOfStream = 5, + /// A packet with profiling info. + ProfileInfo = 6, + /// A block of data with totals (compressed or not). + Totals = 7, + /// A block of data with minimums and maximums (compressed or not). + Extremes = 8, + }; +} - /// То, что передаёт клиент. - namespace ClientCodes { - enum { - Hello = 0, /// Имя, версия, ревизия, БД по-умолчанию. - Query = 1, /** Идентификатор запроса, настройки на отдельный запрос, - * информация, до какой стадии исполнять запрос, - * использовать ли сжатие, текст запроса (без данных для INSERT-а). - */ - Data = 2, /// Блок данных со сжатием или без. - Cancel = 3, /// Отменить выполнение запроса. - Ping = 4, /// Проверка живости соединения с сервером. - }; - } +/// Packet types that client transmits. +namespace ClientCodes { + enum { + /// Name, version, revision, default DB. + Hello = 0, + /// Query id, query settings, stage up to which the query must be executed, + /// whether the compression must be used, + /// query text (without data for INSERTs). + Query = 1, + /// A block of data (compressed or not). + Data = 2, + /// Cancel the query execution. + Cancel = 3, + /// Check that the connection to the server is alive. + Ping = 4, + }; +} - /// Использовать ли сжатие. - namespace CompressionState { - enum { - Disable = 0, - Enable = 1, - }; - } +/// Whether the compression must be used. +namespace CompressionState { + enum { + Disable = 0, + Enable = 1, + }; +} - namespace Stages { - enum { - Complete = 2, - }; - } +namespace Stages { + enum { + Complete = 2, + }; } + +} // namespace clickhouse diff --git a/lib/clickhouse-cpp/clickhouse/query.cpp b/lib/clickhouse-cpp/clickhouse/query.cpp index 89a9037..82307af 100644 --- a/lib/clickhouse-cpp/clickhouse/query.cpp +++ b/lib/clickhouse-cpp/clickhouse/query.cpp @@ -18,4 +18,49 @@ Query::Query(const std::string& query) Query::~Query() { } +void Query::OnData(const Block& block) { + if (select_cb_) { + select_cb_(block); + } } + +bool Query::OnDataCancelable(const Block& block) { + if (select_cancelable_cb_) { + return select_cancelable_cb_(block); + } else { + return true; + } +} + +void Query::OnExtremes(const Block& block) { + if (extremes_cb_) { + extremes_cb_(block); + } +} + +void Query::OnServerException(const Exception& e) { + if (exception_cb_) { + exception_cb_(e); + } +} + +void Query::OnProfile(const Profile& profile) { + (void)profile; +} + +void Query::OnProgress(const Progress& progress) { + if (progress_cb_) { + progress_cb_(progress); + } +} + +void Query::OnFinish() { +} + +void Query::OnTotals(const Block& block) { + if (totals_cb_) { + totals_cb_(block); + } +} + +} // namespace clickhouse diff --git a/lib/clickhouse-cpp/clickhouse/query.h b/lib/clickhouse-cpp/clickhouse/query.h index c9d7e68..60878d0 100644 --- a/lib/clickhouse-cpp/clickhouse/query.h +++ b/lib/clickhouse-cpp/clickhouse/query.h @@ -13,11 +13,11 @@ namespace clickhouse { * Settings of individual query. */ struct QuerySettings { - /// Максимальное количество потоков выполнения запроса. По-умолчанию - определять автоматически. + /// A maximum number of threads for query execution. Determined automatically by default. int max_threads = 0; - /// Считать минимумы и максимумы столбцов результата. + /// Calculate minimum and maximum values of each column. bool extremes = false; - /// Тихо пропускать недоступные шарды. + /// Silently skip unavailable shards. bool skip_unavailable_shards = false; /// Write statistics about read rows, bytes, time elapsed, etc. bool output_format_write_statistics = true; @@ -32,7 +32,6 @@ struct QuerySettings { // priority = 0 }; - struct Exception { int code = 0; std::string name; @@ -42,7 +41,6 @@ struct Exception { std::unique_ptr nested; }; - struct Profile { uint64_t rows = 0; uint64_t blocks = 0; @@ -52,23 +50,23 @@ struct Profile { bool calculated_rows_before_limit = false; }; - struct Progress { uint64_t rows = 0; uint64_t bytes = 0; uint64_t total_rows = 0; }; - class QueryEvents { public: - virtual ~QueryEvents() - { } + virtual ~QueryEvents() = default; - /// Some data was received. + /// Some data has been received. virtual void OnData(const Block& block) = 0; virtual bool OnDataCancelable(const Block& block) = 0; + /// A block with extremes values has been received. + virtual void OnExtremes(const Block& block) = 0; + virtual void OnServerException(const Exception& e) = 0; virtual void OnProfile(const Profile& profile) = 0; @@ -76,21 +74,23 @@ class QueryEvents { virtual void OnProgress(const Progress& progress) = 0; virtual void OnFinish() = 0; -}; + /// A block with totals values has been received. + virtual void OnTotals(const Block& block) = 0; +}; -using ExceptionCallback = std::function; -using ProgressCallback = std::function; -using SelectCallback = std::function; +using DataCallback = std::function; +using ExceptionCallback = std::function; +using ProgressCallback = std::function; +using SelectCallback = DataCallback; using SelectCancelableCallback = std::function; - class Query : public QueryEvents { public: Query(); Query(const char* query); Query(const std::string& query); - ~Query(); + ~Query() override; /// inline std::string GetText() const { @@ -98,7 +98,7 @@ class Query : public QueryEvents { } /// Set handler for receiving result data. - inline Query& OnData(SelectCallback cb) { + inline Query& OnData(DataCallback cb) { select_cb_ = cb; return *this; } @@ -114,6 +114,11 @@ class Query : public QueryEvents { return *this; } + /// Set handler for receiving extremes values. + inline Query& OnExtremes(DataCallback cb) { + extremes_cb_ = cb; + return *this; + } /// Set handler for receiving a progress of query exceution. inline Query& OnProgress(ProgressCallback cb) { @@ -121,46 +126,37 @@ class Query : public QueryEvents { return *this; } -private: - void OnData(const Block& block) override { - if (select_cb_) { - select_cb_(block); - } + /// Set handler for receiving totals values. + inline Query& OnTotals(DataCallback cb) { + totals_cb_ = cb; + return *this; } - bool OnDataCancelable(const Block& block) override { - if (select_cancelable_cb_) { - return select_cancelable_cb_(block); - } else { - return true; - } - } +private: + void OnData(const Block& block) override; - void OnServerException(const Exception& e) override { - if (exception_cb_) { - exception_cb_(e); - } - } + bool OnDataCancelable(const Block& block) override; - void OnProfile(const Profile& profile) override { - (void)profile; - } + void OnExtremes(const Block& block) override; - void OnProgress(const Progress& progress) override { - if (progress_cb_) { - progress_cb_(progress); - } - } + void OnServerException(const Exception& e) override; - void OnFinish() override { - } + void OnProfile(const Profile& profile) override; + + void OnProgress(const Progress& progress) override; + + void OnFinish() override; + + void OnTotals(const Block& block) override; private: std::string query_; ExceptionCallback exception_cb_; ProgressCallback progress_cb_; - SelectCallback select_cb_; + DataCallback select_cb_; SelectCancelableCallback select_cancelable_cb_; + DataCallback totals_cb_; + DataCallback extremes_cb_; }; -} +} // namespace clickhouse diff --git a/lib/clickhouse-cpp/clickhouse/types/type_parser.cpp b/lib/clickhouse-cpp/clickhouse/types/type_parser.cpp index 0b1fa20..7b1c997 100644 --- a/lib/clickhouse-cpp/clickhouse/types/type_parser.cpp +++ b/lib/clickhouse-cpp/clickhouse/types/type_parser.cpp @@ -1,6 +1,8 @@ #include "type_parser.h" #include "../base/string_utils.h" +#include +#include #include namespace clickhouse { @@ -26,6 +28,12 @@ static const std::unordered_map kTypeCode = { { "Enum8", Type::Enum8 }, { "Enum16", Type::Enum16 }, { "UUID", Type::UUID }, + { "IPv4", Type::IPv4 }, + { "IPv6", Type::IPv6 }, + { "Decimal", Type::Decimal }, + { "Decimal32", Type::Decimal32 }, + { "Decimal64", Type::Decimal64 }, + { "Decimal128", Type::Decimal128 }, }; static Type::Code GetTypeCode(const std::string& name) { @@ -167,8 +175,10 @@ const TypeAst* ParseTypeName(const std::string& type_name) { // Cache for type_name. // Usually we won't have too many type names in the cache, so do not try to // limit cache size. - static std::unordered_map ast_cache; + static std::map ast_cache; + static std::mutex lock; + std::lock_guard guard(lock); auto it = ast_cache.find(type_name); if (it != ast_cache.end()) { return &it->second; diff --git a/lib/clickhouse-cpp/clickhouse/types/type_parser.h b/lib/clickhouse-cpp/clickhouse/types/type_parser.h index 31c0e1c..fa1d708 100644 --- a/lib/clickhouse-cpp/clickhouse/types/type_parser.h +++ b/lib/clickhouse-cpp/clickhouse/types/type_parser.h @@ -17,7 +17,7 @@ struct TypeAst { Number, Terminal, Tuple, - Enum + Enum, }; /// Type's category. @@ -31,7 +31,7 @@ struct TypeAst { int64_t value = 0; /// Subelements of the type. /// Used to store enum's names and values as well. - std::list elements; + std::vector elements; }; diff --git a/lib/clickhouse-cpp/clickhouse/types/types.cpp b/lib/clickhouse-cpp/clickhouse/types/types.cpp index 09b815c..591c07b 100644 --- a/lib/clickhouse-cpp/clickhouse/types/types.cpp +++ b/lib/clickhouse-cpp/clickhouse/types/types.cpp @@ -15,6 +15,8 @@ Type::Type(const Code code) nullable_ = new NullableImpl; } else if (code_ == Enum8 || code_ == Enum16) { enum_ = new EnumImpl; + } else if (code_== Decimal || code_== Decimal32 || code_ == Decimal64 || code_ == Decimal128) { + decimal_ = new DecimalImpl; } } @@ -27,6 +29,8 @@ Type::~Type() { delete nullable_; } else if (code_ == Enum8 || code_ == Enum16) { delete enum_; + } else if (code_== Decimal || code_== Decimal32 || code_ == Decimal64 || code_ == Decimal128) { + delete decimal_; } } @@ -49,7 +53,10 @@ TypeRef Type::GetNestedType() const { } std::vector Type::GetTupleType() const { - return tuple_->item_types; + if (code_ == Tuple) { + return tuple_->item_types; + } + return std::vector(); } std::string Type::GetName() const { @@ -64,6 +71,8 @@ std::string Type::GetName() const { return "Int32"; case Int64: return "Int64"; + case Int128: + return "Int128"; case UInt8: return "UInt8"; case UInt16: @@ -82,6 +91,10 @@ std::string Type::GetName() const { return "String"; case FixedString: return "FixedString(" + std::to_string(string_size_) + ")"; + case IPv4: + return "IPv4"; + case IPv6: + return "IPv6"; case DateTime: return "DateTime"; case Date: @@ -125,6 +138,14 @@ std::string Type::GetName() const { result += ")"; return result; } + case Decimal: + return "Decimal(" + std::to_string(decimal_->precision) + "," + std::to_string(decimal_->scale) + ")"; + case Decimal32: + return "Decimal32(" + std::to_string(decimal_->scale) + ")"; + case Decimal64: + return "Decimal64(" + std::to_string(decimal_->scale) + ")"; + case Decimal128: + return "Decimal128(" + std::to_string(decimal_->scale) + ")"; } return std::string(); @@ -148,6 +169,25 @@ TypeRef Type::CreateDateTime() { return TypeRef(new Type(Type::DateTime)); } +TypeRef Type::CreateDecimal(size_t precision, size_t scale) { + TypeRef type(new Type(Type::Decimal)); + type->decimal_->precision = precision; + type->decimal_->scale = scale; + return type; +} + +TypeRef Type::CreateIPv4() { + return TypeRef(new Type(Type::IPv4)); +} + +TypeRef Type::CreateIPv6() { + return TypeRef(new Type(Type::IPv6)); +} + +TypeRef Type::CreateNothing() { + return TypeRef(new Type(Type::Void)); +} + TypeRef Type::CreateNullable(TypeRef nested_type) { TypeRef type(new Type(Type::Nullable)); type->nullable_->nested_type = nested_type; diff --git a/lib/clickhouse-cpp/clickhouse/types/types.h b/lib/clickhouse-cpp/clickhouse/types/types.h index e8d6c99..a427f1f 100644 --- a/lib/clickhouse-cpp/clickhouse/types/types.h +++ b/lib/clickhouse-cpp/clickhouse/types/types.h @@ -33,6 +33,13 @@ class Type { Enum8, Enum16, UUID, + IPv4, + IPv6, + Int128, + Decimal, + Decimal32, + Decimal64, + Decimal128, }; struct EnumItem { @@ -68,6 +75,14 @@ class Type { static TypeRef CreateDateTime(); + static TypeRef CreateDecimal(size_t precision, size_t scale); + + static TypeRef CreateIPv4(); + + static TypeRef CreateIPv6(); + + static TypeRef CreateNothing(); + static TypeRef CreateNullable(TypeRef nested_type); template @@ -92,6 +107,11 @@ class Type { TypeRef item_type; }; + struct DecimalImpl { + size_t precision; + size_t scale; + }; + struct NullableImpl { TypeRef nested_type; }; @@ -113,6 +133,7 @@ class Type { const Code code_; union { ArrayImpl* array_; + DecimalImpl* decimal_; NullableImpl* nullable_; TupleImpl* tuple_; EnumImpl* enum_; @@ -162,6 +183,11 @@ inline TypeRef Type::CreateSimple() { return TypeRef(new Type(Int64)); } +template <> +inline TypeRef Type::CreateSimple<__int128>() { + return TypeRef(new Type(Int128)); +} + template <> inline TypeRef Type::CreateSimple() { return TypeRef(new Type(UInt8)); diff --git a/lib/clickhouse-cpp/cmake/cpp11.cmake b/lib/clickhouse-cpp/cmake/cpp11.cmake deleted file mode 100644 index bd47583..0000000 --- a/lib/clickhouse-cpp/cmake/cpp11.cmake +++ /dev/null @@ -1,8 +0,0 @@ -MACRO (USE_CXX11) - IF (CMAKE_VERSION VERSION_LESS "3.1") - SET (CMAKE_CXX_FLAGS "-std=c++11 ${CMAKE_CXX_FLAGS}") - ELSE () - SET (CMAKE_CXX_STANDARD 11) - SET (CMAKE_CXX_STANDARD_REQUIRED ON) - ENDIF () -ENDMACRO (USE_CXX11) diff --git a/lib/clickhouse-cpp/cmake/cpp17.cmake b/lib/clickhouse-cpp/cmake/cpp17.cmake new file mode 100644 index 0000000..37edc55 --- /dev/null +++ b/lib/clickhouse-cpp/cmake/cpp17.cmake @@ -0,0 +1,8 @@ +MACRO (USE_CXX17) + IF (CMAKE_VERSION VERSION_LESS "3.1") + SET (CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++17") + ELSE () + SET (CMAKE_CXX_STANDARD 17) + SET (CMAKE_CXX_STANDARD_REQUIRED ON) + ENDIF () +ENDMACRO (USE_CXX17) diff --git a/lib/clickhouse-cpp/contrib/cityhash/BUCK b/lib/clickhouse-cpp/contrib/cityhash/BUCK deleted file mode 100644 index 2d14bbb..0000000 --- a/lib/clickhouse-cpp/contrib/cityhash/BUCK +++ /dev/null @@ -1,13 +0,0 @@ -cxx_library( - name = 'cityhash', - header_namespace = 'cityhash', - exported_headers = subdir_glob([ - ('', '*.h'), - ]), - srcs = glob([ - '*.cc', - ]), - visibility = [ - '//...', - ], -) diff --git a/lib/clickhouse-cpp/contrib/gtest/BUCK b/lib/clickhouse-cpp/contrib/gtest/BUCK deleted file mode 100644 index d888dbe..0000000 --- a/lib/clickhouse-cpp/contrib/gtest/BUCK +++ /dev/null @@ -1,14 +0,0 @@ -cxx_library( - name = 'gtest', - # This is a bit weird, but this is how the tests use the headers... - header_namespace = 'contrib/gtest', - exported_headers = subdir_glob([ - ('', '*.h'), - ]), - srcs = glob([ - '*.cc', - ]), - visibility = [ - '//...', - ], -) diff --git a/lib/clickhouse-cpp/contrib/lz4/BUCK b/lib/clickhouse-cpp/contrib/lz4/BUCK deleted file mode 100644 index 0802605..0000000 --- a/lib/clickhouse-cpp/contrib/lz4/BUCK +++ /dev/null @@ -1,13 +0,0 @@ -cxx_library( - name = 'lz4', - header_namespace = 'lz4', - exported_headers = subdir_glob([ - ('', '*.h'), - ]), - srcs = glob([ - '*.c', - ]), - visibility = [ - '//...', - ], -) diff --git a/lib/clickhouse-cpp/tests/simple/BUCK b/lib/clickhouse-cpp/tests/simple/BUCK deleted file mode 100644 index 37bf2ce..0000000 --- a/lib/clickhouse-cpp/tests/simple/BUCK +++ /dev/null @@ -1,12 +0,0 @@ -cxx_binary( - name = 'simple', - srcs = [ - 'main.cpp', - ], - compiler_flags = [ - '-std=c++11', - ], - deps = [ - '//:clickhouse-cpp', - ], -) diff --git a/lib/clickhouse-cpp/tests/simple/main.cpp b/lib/clickhouse-cpp/tests/simple/main.cpp index 2c7b942..2e57704 100644 --- a/lib/clickhouse-cpp/tests/simple/main.cpp +++ b/lib/clickhouse-cpp/tests/simple/main.cpp @@ -47,7 +47,6 @@ inline void ArrayExample(Client& client) { b.AppendColumn("arr", arr); client.Insert("test.array", b); - client.Select("SELECT arr FROM test.array", [](const Block& block) { for (size_t c = 0; c < block.GetRowCount(); ++c) { @@ -64,6 +63,46 @@ inline void ArrayExample(Client& client) { client.Execute("DROP TABLE test.array"); } +inline void MultiArrayExample(Client& client) { + Block b; + + /// Create a table. + client.Execute("CREATE TABLE IF NOT EXISTS test.multiarray (arr Array(Array(UInt64))) ENGINE = Memory"); + + auto arr = std::make_shared(std::make_shared()); + + auto id = std::make_shared(); + id->Append(17); + arr->AppendAsColumn(id); + + auto a2 = std::make_shared(std::make_shared(std::make_shared())); + a2->AppendAsColumn(arr); + b.AppendColumn("arr", a2); + client.Insert("test.multiarray", b); + + client.Select("SELECT arr FROM test.multiarray", [](const Block& block) + { + for (size_t c = 0; c < block.GetRowCount(); ++c) { + auto col = block[0]->As()->GetAsColumn(c); + cout << "["; + for (size_t i = 0; i < col->Size(); ++i) { + auto col2 = col->As()->GetAsColumn(i); + for (size_t j = 0; j < col2->Size(); ++j) { + cout << (int)(*col2->As())[j]; + if (j + 1 != col2->Size()) { + cout << " "; + } + } + } + std::cout << "]" << std::endl; + } + } + ); + + /// Delete table. + client.Execute("DROP TABLE test.multiarray"); +} + inline void DateExample(Client& client) { Block b; @@ -75,7 +114,6 @@ inline void DateExample(Client& client) { b.AppendColumn("d", d); client.Insert("test.date", b); - client.Select("SELECT d FROM test.date", [](const Block& block) { for (size_t c = 0; c < block.GetRowCount(); ++c) { @@ -90,6 +128,39 @@ inline void DateExample(Client& client) { client.Execute("DROP TABLE test.date"); } +inline void DecimalExample(Client& client) { + Block b; + + /// Create a table. + client.Execute("CREATE TABLE IF NOT EXISTS test.decimal (d Decimal64(4)) ENGINE = Memory"); + + auto d = std::make_shared(18, 4); + d->Append(21111); + b.AppendColumn("d", d); + client.Insert("test.decimal", b); + + client.Select("SELECT d FROM test.decimal", [](const Block& block) + { + for (size_t c = 0; c < block.GetRowCount(); ++c) { + auto col = block[0]->As(); + cout << (int)col->At(c) << endl; + } + } + ); + + client.Select("SELECT toDecimal32(2, 4) AS x", [](const Block& block) + { + for (size_t c = 0; c < block.GetRowCount(); ++c) { + auto col = block[0]->As(); + cout << (int)col->At(c) << endl; + } + } + ); + + /// Delete table. + client.Execute("DROP TABLE test.decimal"); +} + inline void GenericExample(Client& client) { /// Create a table. client.Execute("CREATE TABLE IF NOT EXISTS test.client (id UInt64, name String) ENGINE = Memory"); @@ -292,20 +363,114 @@ inline void EnumExample(Client& client) { } ); - /// Delete table. client.Execute("DROP TABLE test.enums"); } +inline void SelectNull(Client& client) { + client.Select("SELECT NULL", [](const Block& block) + { + assert(block.GetRowCount() < 2); + } + ); +} + +inline void ShowTables(Client& client) { + /// Select values inserted in the previous step. + client.Select("SHOW TABLES", [](const Block& block) + { + for (size_t i = 0; i < block.GetRowCount(); ++i) { + std::cout << (*block[0]->As())[i] << "\n"; + } + } + ); +} + +inline void IPExample(Client &client) { + /// Create a table. + client.Execute("CREATE TABLE IF NOT EXISTS test.ips (id UInt64, v4 IPv4, v6 IPv6) ENGINE = Memory"); + + /// Insert some values. + { + Block block; + + auto id = std::make_shared(); + id->Append(1); + id->Append(2); + id->Append(3); + + auto v4 = std::make_shared(); + v4->Append("127.0.0.1"); + v4->Append(3585395774); + v4->Append(0); + + auto v6 = std::make_shared(); + v6->Append("::1"); + v6->Append("aa::ff"); + v6->Append("fe80::86ba:ef31:f2d8:7e8b"); + + block.AppendColumn("id", id); + block.AppendColumn("v4", v4); + block.AppendColumn("v6", v6); + + client.Insert("test.ips", block); + } + + /// Select values inserted in the previous step. + client.Select("SELECT id, v4, v6 FROM test.ips", [](const Block& block) + { + for (Block::Iterator bi(block); bi.IsValid(); bi.Next()) { + std::cout << bi.Name() << " "; + } + std::cout << std::endl; + + for (size_t i = 0; i < block.GetRowCount(); ++i) { + std::cout << (*block[0]->As())[i] << " " + << (*block[1]->As()).AsString(i) << " (" << (*block[1]->As())[i].s_addr << ") " + << (*block[2]->As()).AsString(i) << "\n"; + } + } + ); + + /// Delete table. + client.Execute("DROP TABLE test.ips"); +} + +void WithTotalsExample(Client& client) { + auto print_data = [](const Block& block) { + if (!block.GetRowCount()) { + return; + } + for (Block::Iterator bi(block); bi.IsValid(); bi.Next()) { + std::cout << bi.Name() << " "; + } + std::cout << std::endl; + for (size_t i = 0; i < block.GetRowCount(); ++i) { + std::cout << (*block[0]->As())[i] << "\n"; + } + }; + + client.Select( + Query("SELECT count(*) FROM system.tables GROUP BY database WITH TOTALS") + .OnData(print_data) + .OnTotals(print_data)); +} + static void RunTests(Client& client) { ArrayExample(client); + CancelableExample(client); DateExample(client); + DecimalExample(client); + EnumExample(client); + ExecptionExample(client); GenericExample(client); + IPExample(client); + MultiArrayExample(client); NullableExample(client); NumbersExample(client); - CancelableExample(client); - ExecptionExample(client); - EnumExample(client); + SelectNull(client); + ShowTables(client); + WithTotalsExample(client); } int main() { diff --git a/lib/clickhouse-cpp/ut/BUCK b/lib/clickhouse-cpp/ut/BUCK deleted file mode 100644 index 136716c..0000000 --- a/lib/clickhouse-cpp/ut/BUCK +++ /dev/null @@ -1,12 +0,0 @@ -cxx_test( - name = 'ut', - srcs = glob([ - '*.cpp', - ]), - compiler_flags = [ - '-std=c++11', - ], - deps = [ - '//:clickhouse-cpp', - ], -) diff --git a/lib/clickhouse-cpp/ut/CMakeLists.txt b/lib/clickhouse-cpp/ut/CMakeLists.txt index 85c6670..1fa66f1 100644 --- a/lib/clickhouse-cpp/ut/CMakeLists.txt +++ b/lib/clickhouse-cpp/ut/CMakeLists.txt @@ -1,10 +1,13 @@ ADD_EXECUTABLE (clickhouse-cpp-ut main.cpp + client_ut.cpp columns_ut.cpp - types_ut.cpp + socket_ut.cpp + stream_ut.cpp + tcp_server.cpp type_parser_ut.cpp - client_ut.cpp + types_ut.cpp ) TARGET_LINK_LIBRARIES (clickhouse-cpp-ut diff --git a/lib/clickhouse-cpp/ut/columns_ut.cpp b/lib/clickhouse-cpp/ut/columns_ut.cpp index a874cf6..0aabee0 100644 --- a/lib/clickhouse-cpp/ut/columns_ut.cpp +++ b/lib/clickhouse-cpp/ut/columns_ut.cpp @@ -37,7 +37,6 @@ static std::vector MakeUUIDs() { 0x3507213c178649f9llu, 0x9faf035d662f60aellu}; } - TEST(ColumnsCase, NumericInit) { auto col = std::make_shared(MakeNumbers()); @@ -110,6 +109,15 @@ TEST(ColumnsCase, DateAppend) { ASSERT_EQ(col2->At(0), (now / 86400) * 86400); } +TEST(ColumnsCase, Date2038) { + auto col1 = std::make_shared(); + std::time_t largeDate(25882ul * 86400ul); + col1->Append(largeDate); + + ASSERT_EQ(col1->Size(), 1u); + ASSERT_EQ(static_cast(col1->At(0)), 25882ul * 86400ul); +} + TEST(ColumnsCase, EnumTest) { std::vector enum_items = {{"Hi", 1}, {"Hello", 2}}; diff --git a/lib/clickhouse-cpp/ut/socket_ut.cpp b/lib/clickhouse-cpp/ut/socket_ut.cpp new file mode 100644 index 0000000..7ffeb14 --- /dev/null +++ b/lib/clickhouse-cpp/ut/socket_ut.cpp @@ -0,0 +1,32 @@ +#include "tcp_server.h" + +#include +#include + +#include +#include +#include +#include + +using namespace clickhouse; + +TEST(Socketcase, connecterror) { + int port = 9978; + NetworkAddress addr("localhost", std::to_string(port)); + LocalTcpServer server(port); + server.start(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + try { + SocketConnect(addr); + } catch (const std::system_error& e) { + FAIL(); + } + std::this_thread::sleep_for(std::chrono::seconds(1)); + server.stop(); + try { + SocketConnect(addr); + FAIL(); + } catch (const std::system_error& e) { + ASSERT_NE(EINPROGRESS,e.code().value()); + } +} diff --git a/lib/clickhouse-cpp/ut/stream_ut.cpp b/lib/clickhouse-cpp/ut/stream_ut.cpp new file mode 100644 index 0000000..b27efb2 --- /dev/null +++ b/lib/clickhouse-cpp/ut/stream_ut.cpp @@ -0,0 +1,23 @@ +#include +#include + +using namespace clickhouse; + +TEST(CodedStreamCase, Varint64) { + Buffer buf; + + { + BufferOutput output(&buf); + CodedOutputStream coded(&output); + coded.WriteVarint64(18446744071965638648ULL); + } + + + { + ArrayInput input(buf.data(), buf.size()); + CodedInputStream coded(&input); + uint64_t value; + ASSERT_TRUE(coded.ReadVarint64(&value)); + ASSERT_EQ(value, 18446744071965638648ULL); + } +} diff --git a/lib/clickhouse-cpp/ut/tcp_server.cpp b/lib/clickhouse-cpp/ut/tcp_server.cpp new file mode 100644 index 0000000..cd47944 --- /dev/null +++ b/lib/clickhouse-cpp/ut/tcp_server.cpp @@ -0,0 +1,56 @@ +#include "tcp_server.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace clickhouse { + +LocalTcpServer::LocalTcpServer(int port) + : port_(port) + , serverSd_(-1) +{} + +LocalTcpServer::~LocalTcpServer() { + stop(); +} + +void LocalTcpServer::start() { + //setup a socket + sockaddr_in servAddr; + bzero((char*)&servAddr, sizeof(servAddr)); + servAddr.sin_family = AF_INET; + servAddr.sin_addr.s_addr = htonl(INADDR_ANY); + servAddr.sin_port = htons(port_); + serverSd_ = socket(AF_INET, SOCK_STREAM, 0); + if (serverSd_ < 0) { + std::cerr << "Error establishing server socket" << std::endl; + throw std::runtime_error("Error establishing server socket"); + } + int enable = 1; + if (setsockopt(serverSd_, SOL_SOCKET, SO_REUSEADDR, &enable, sizeof(int)) < 0) { + std::cerr << "setsockopt(SO_REUSEADDR) failed" << std::endl; + } + //bind the socket to its local address + int bindStatus = bind(serverSd_, (struct sockaddr*) &servAddr, sizeof(servAddr)); + if (bindStatus < 0) { + std::cerr << "Error binding socket to local address" << std::endl; + throw std::runtime_error("Error binding socket to local address"); + } + listen(serverSd_, 3); +} + +void LocalTcpServer::stop() { + if(serverSd_ > 0) { + shutdown(serverSd_, SHUT_RDWR); + close(serverSd_); + serverSd_ = -1; + } +} + +} diff --git a/lib/clickhouse-cpp/ut/tcp_server.h b/lib/clickhouse-cpp/ut/tcp_server.h new file mode 100644 index 0000000..d8f3cd5 --- /dev/null +++ b/lib/clickhouse-cpp/ut/tcp_server.h @@ -0,0 +1,21 @@ +#pragma once + +namespace clickhouse { + +class LocalTcpServer { +public: + LocalTcpServer(int port); + ~LocalTcpServer(); + + void start(); + void stop(); + +private: + void startImpl(); + +private: + int port_; + int serverSd_; +}; + +} diff --git a/lib/clickhouse-cpp/ut/type_parser_ut.cpp b/lib/clickhouse-cpp/ut/type_parser_ut.cpp index 26b51b2..25eaaa9 100644 --- a/lib/clickhouse-cpp/ut/type_parser_ut.cpp +++ b/lib/clickhouse-cpp/ut/type_parser_ut.cpp @@ -65,3 +65,63 @@ TEST(TypeParserCase, ParseEnum) { ++element; } } + +TEST(TypeParserCase, ParseTuple) { + TypeAst ast; + TypeParser( + "Tuple(UInt8, String)") + .Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Tuple); + ASSERT_EQ(ast.name, "Tuple"); + ASSERT_EQ(ast.code, Type::Tuple); + ASSERT_EQ(ast.elements.size(), 2u); + + std::vector names = {"UInt8", "String"}; + + auto element = ast.elements.begin(); + for (size_t i = 0; i < 2; ++i) { + ASSERT_EQ(element->name, names[i]); + ++element; + } +} + +TEST(TypeParserCase, ParseDecimal) { + TypeAst ast; + TypeParser("Decimal(12, 5)").Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Terminal); + ASSERT_EQ(ast.name, "Decimal"); + ASSERT_EQ(ast.code, Type::Decimal); + ASSERT_EQ(ast.elements.size(), 2u); + ASSERT_EQ(ast.elements[0].value, 12); + ASSERT_EQ(ast.elements[1].value, 5); +} + +TEST(TypeParserCase, ParseDecimal32) { + TypeAst ast; + TypeParser("Decimal32(7)").Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Terminal); + ASSERT_EQ(ast.name, "Decimal32"); + ASSERT_EQ(ast.code, Type::Decimal32); + ASSERT_EQ(ast.elements.size(), 1u); + ASSERT_EQ(ast.elements[0].value, 7); +} + +TEST(TypeParserCase, ParseDecimal64) { + TypeAst ast; + TypeParser("Decimal64(1)").Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Terminal); + ASSERT_EQ(ast.name, "Decimal64"); + ASSERT_EQ(ast.code, Type::Decimal64); + ASSERT_EQ(ast.elements.size(), 1u); + ASSERT_EQ(ast.elements[0].value, 1); +} + +TEST(TypeParserCase, ParseDecimal128) { + TypeAst ast; + TypeParser("Decimal128(3)").Parse(&ast); + ASSERT_EQ(ast.meta, TypeAst::Terminal); + ASSERT_EQ(ast.name, "Decimal128"); + ASSERT_EQ(ast.code, Type::Decimal128); + ASSERT_EQ(ast.elements.size(), 1u); + ASSERT_EQ(ast.elements[0].value, 3); +} From 776ea002f197f4330ab9af82f2f82491a3505f9b Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 15 Apr 2020 14:02:22 -0400 Subject: [PATCH 09/16] timeouts --- SeasClick.cpp | 51 +++++++++++++++--- config.m4 | 3 ++ lib/clickhouse-cpp/clickhouse/base/socket.cpp | 9 +++- lib/clickhouse-cpp/clickhouse/base/socket.h | 3 +- lib/clickhouse-cpp/clickhouse/client.cpp | 53 ++++++++++++++++++- lib/clickhouse-cpp/clickhouse/client.h | 9 ++++ typesToPhp.cpp | 4 +- 7 files changed, 118 insertions(+), 14 deletions(-) diff --git a/SeasClick.cpp b/SeasClick.cpp index 205d57e..ec775ec 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -61,7 +61,7 @@ static PHP_METHOD(SEASCLICK_RES_NAME, insert); static PHP_METHOD(SEASCLICK_RES_NAME, execute); ZEND_BEGIN_ARG_INFO_EX(SeasCilck_construct, 0, 0, 1) -ZEND_ARG_INFO(0, connectParames) +ZEND_ARG_INFO(0, connectParams) ZEND_END_ARG_INFO() ZEND_BEGIN_ARG_INFO_EX(SeasCilck_select, 0, 0, 3) @@ -124,7 +124,10 @@ PHP_MINIT_FUNCTION(SeasClick) zend_declare_property_null(SeasClick_ce, "user", strlen("user"), ZEND_ACC_PROTECTED TSRMLS_CC); zend_declare_property_null(SeasClick_ce, "passwd", strlen("passwd"), ZEND_ACC_PROTECTED TSRMLS_CC); zend_declare_property_bool(SeasClick_ce, "compression", strlen("compression"), false, ZEND_ACC_PROTECTED TSRMLS_CC); - + zend_declare_property_long(SeasClick_ce, "retry_timeout", strlen("retry_timeout"), 5, ZEND_ACC_PROTECTED TSRMLS_CC); + zend_declare_property_long(SeasClick_ce, "retry_count", strlen("retry_count"), 1, ZEND_ACC_PROTECTED TSRMLS_CC); + zend_declare_property_long(SeasClick_ce, "receive_timeout", strlen("receive_timeout"), 0, ZEND_ACC_PROTECTED TSRMLS_CC); + zend_declare_property_long(SeasClick_ce, "connect_timeout", strlen("connect_timeout"), 5, ZEND_ACC_PROTECTED TSRMLS_CC); REGISTER_SC_CLASS_CONST_LONG("FETCH_ONE", (zend_long)SC_FETCH_ONE); REGISTER_SC_CLASS_CONST_LONG("FETCH_KEY_PAIR", (zend_long)SC_FETCH_KEY_PAIR); @@ -167,14 +170,14 @@ zend_module_entry SeasClick_module_entry = }; /* }}} */ -/* {{{ proto object __construct(array connectParames) +/* {{{ proto object __construct(array connectParams) */ PHP_METHOD(SEASCLICK_RES_NAME, __construct) { - zval *connectParames; + zval *connectParams; #ifndef FAST_ZPP - if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z", &connectParames) == FAILURE) + if (zend_parse_parameters(ZEND_NUM_ARGS() TSRMLS_CC, "z", &connectParams) == FAILURE) { return; } @@ -182,13 +185,13 @@ PHP_METHOD(SEASCLICK_RES_NAME, __construct) #undef IS_UNDEF #define IS_UNDEF Z_EXPECTED_LONG ZEND_PARSE_PARAMETERS_START(1, 1) - Z_PARAM_ARRAY(connectParames) + Z_PARAM_ARRAY(connectParams) ZEND_PARSE_PARAMETERS_END(); #undef IS_UNDEF #define IS_UNDEF 0 #endif - HashTable *_ht = Z_ARRVAL_P(connectParames); + HashTable *_ht = Z_ARRVAL_P(connectParams); zval *value; zval *this_obj; @@ -211,15 +214,47 @@ PHP_METHOD(SEASCLICK_RES_NAME, __construct) zend_update_property_bool(SeasClick_ce, this_obj, "compression", sizeof("compression") - 1, Z_LVAL_P(value) TSRMLS_CC); } + if (php_array_get_value(_ht, "retry_timeout", value)) + { + convert_to_long(value); + zend_update_property_long(SeasClick_ce, this_obj, "retry_timeout", sizeof("retry_timeout") - 1, Z_LVAL_P(value) TSRMLS_CC); + } + + if (php_array_get_value(_ht, "retry_count", value)) + { + convert_to_long(value); + zend_update_property_long(SeasClick_ce, this_obj, "retry_count", sizeof("retry_count") - 1, Z_LVAL_P(value) TSRMLS_CC); + } + + if (php_array_get_value(_ht, "connect_timeout", value)) + { + convert_to_long(value); + zend_update_property_long(SeasClick_ce, this_obj, "connect_timeout", sizeof("connect_timeout") - 1, Z_LVAL_P(value) TSRMLS_CC); + } + + if (php_array_get_value(_ht, "receive_timeout", value)) + { + convert_to_long(value); + zend_update_property_long(SeasClick_ce, this_obj, "receive_timeout", sizeof("receive_timeout") - 1, Z_LVAL_P(value) TSRMLS_CC); + } + zval *host = sc_zend_read_property(SeasClick_ce, this_obj, "host", sizeof("host") - 1, 0); zval *port = sc_zend_read_property(SeasClick_ce, this_obj, "port", sizeof("port") - 1, 0); zval *compression = sc_zend_read_property(SeasClick_ce, this_obj, "compression", sizeof("compression") - 1, 0); + zval *retry_timeout = sc_zend_read_property(SeasClick_ce, this_obj, "retry_timeout", sizeof("retry_timeout") - 1, 0); + zval *retry_count = sc_zend_read_property(SeasClick_ce, this_obj, "retry_count", sizeof("retry_count") - 1, 0); + zval *receive_timeout = sc_zend_read_property(SeasClick_ce, this_obj, "receive_timeout", sizeof("receive_timeout") - 1, 0); + zval *connect_timeout = sc_zend_read_property(SeasClick_ce, this_obj, "connect_timeout", sizeof("connect_timeout") - 1, 0); ClientOptions Options = ClientOptions() .SetHost(Z_STRVAL_P(host)) .SetPort(Z_LVAL_P(port)) + .SetSendRetries(Z_LVAL_P(retry_count)) + .SetRetryTimeout(std::chrono::seconds(Z_LVAL_P(retry_timeout))) + .SetSocketReceiveTimeout(std::chrono::seconds(Z_LVAL_P(receive_timeout))) + .SetSocketConnectTimeout(std::chrono::seconds(Z_LVAL_P(connect_timeout))) .SetPingBeforeQuery(false); - if (Z_TYPE_P(compression) == IS_TRUE) + if (Z_LVAL_P(compression) == 1) { Options = Options.SetCompressionMethod(CompressionMethod::LZ4); } diff --git a/config.m4 b/config.m4 index 485b54b..4e56a42 100644 --- a/config.m4 +++ b/config.m4 @@ -79,6 +79,9 @@ if test "$PHP_SEASCLICK" != "no"; then lib/clickhouse-cpp/clickhouse/columns/string.cpp \ lib/clickhouse-cpp/clickhouse/columns/tuple.cpp \ lib/clickhouse-cpp/clickhouse/columns/uuid.cpp \ + lib/clickhouse-cpp/clickhouse/columns/decimal.cpp \ + lib/clickhouse-cpp/clickhouse/columns/ip4.cpp \ + lib/clickhouse-cpp/clickhouse/columns/ip6.cpp \ lib/clickhouse-cpp/clickhouse/types/type_parser.cpp \ lib/clickhouse-cpp/clickhouse/types/types.cpp \ lib/clickhouse-cpp/contrib/cityhash/city.cc \ diff --git a/lib/clickhouse-cpp/clickhouse/base/socket.cpp b/lib/clickhouse-cpp/clickhouse/base/socket.cpp index b8d4510..846f391 100644 --- a/lib/clickhouse-cpp/clickhouse/base/socket.cpp +++ b/lib/clickhouse-cpp/clickhouse/base/socket.cpp @@ -1,6 +1,7 @@ #include "socket.h" #include "singleton.h" +#include #include #include #include @@ -253,7 +254,7 @@ NetworkInitializer::NetworkInitializer() { } -SOCKET SocketConnect(const NetworkAddress& addr) { +SOCKET SocketConnect(const NetworkAddress& addr, std::chrono::seconds socketReceiveTimeout, std::chrono::seconds socketConnectTimeout) { int last_err = 0; for (auto res = addr.Info(); res != nullptr; res = res->ai_next) { SOCKET s(socket(res->ai_family, res->ai_socktype, res->ai_protocol)); @@ -264,6 +265,10 @@ SOCKET SocketConnect(const NetworkAddress& addr) { SetNonBlock(s, true); + /* Timeout in seconds */ + timeval tv{socketReceiveTimeout.count(), 0}; + setsockopt(s, SOL_SOCKET, SO_RCVTIMEO, &tv , sizeof(tv)); + if (connect(s, res->ai_addr, (int)res->ai_addrlen) != 0) { int err = errno; if (err == EINPROGRESS || err == EAGAIN || err == EWOULDBLOCK) { @@ -271,7 +276,7 @@ SOCKET SocketConnect(const NetworkAddress& addr) { fd.fd = s; fd.events = POLLOUT; fd.revents = 0; - ssize_t rval = Poll(&fd, 1, 5000); + ssize_t rval = Poll(&fd, 1, socketConnectTimeout.count() * 1000); if (rval == -1) { throw std::system_error(errno, std::system_category(), "fail to connect"); diff --git a/lib/clickhouse-cpp/clickhouse/base/socket.h b/lib/clickhouse-cpp/clickhouse/base/socket.h index 2d1bddd..85d4d23 100644 --- a/lib/clickhouse-cpp/clickhouse/base/socket.h +++ b/lib/clickhouse-cpp/clickhouse/base/socket.h @@ -6,6 +6,7 @@ #include #include +#include #if defined(_win_) # pragma comment(lib, "Ws2_32.lib") @@ -106,7 +107,7 @@ static struct NetworkInitializer { } gNetworkInitializer; /// -SOCKET SocketConnect(const NetworkAddress& addr); +SOCKET SocketConnect(const NetworkAddress& addr, std::chrono::seconds socketReceiveTimeout, std::chrono::seconds socketConnectTimeout); ssize_t Poll(struct pollfd* fds, int nfds, int timeout) noexcept; diff --git a/lib/clickhouse-cpp/clickhouse/client.cpp b/lib/clickhouse-cpp/clickhouse/client.cpp index f2c62d1..03169a6 100644 --- a/lib/clickhouse-cpp/clickhouse/client.cpp +++ b/lib/clickhouse-cpp/clickhouse/client.cpp @@ -80,6 +80,10 @@ class Client::Impl { void ResetConnection(); + void InsertQuery(Query query); + + void InsertData(const Block& block); + private: bool Handshake(); @@ -195,6 +199,19 @@ void Client::Impl::ExecuteQuery(Query query) { } } +void Client::Impl::InsertData(const Block& block) { + // Send data. + SendData(block); + // Send empty block as marker of + // end of data. + SendData(Block()); + + // Wait for EOS. + while (ReceivePacket()) { + ; + } +} + void Client::Impl::Insert(const std::string& table_name, const Block& block) { if (options_.ping_before_query) { RetryGuard([this]() { Ping(); }); @@ -261,7 +278,7 @@ void Client::Impl::Ping() { } void Client::Impl::ResetConnection() { - SocketHolder s(SocketConnect(NetworkAddress(options_.host, std::to_string(options_.port)))); + SocketHolder s(SocketConnect(NetworkAddress(options_.host, std::to_string(options_.port)), options_.socket_receive_timeout, options_.socket_connect_timeout)); if (s.Closed()) { throw std::system_error(errno, std::system_category()); @@ -294,6 +311,32 @@ bool Client::Impl::Handshake() { return true; } +void Client::Impl::InsertQuery(Query query) { + EnsureNull en(static_cast(&query), &events_); + + if (options_.ping_before_query) { + RetryGuard([this]() { Ping(); }); + } + + SendQuery(query.GetText()); + + uint64_t server_packet; + // Receive data packet. + while (true) { + bool ret = ReceivePacket(&server_packet); + + if (!ret) { + throw std::runtime_error("fail to receive data packet"); + } + if (server_packet == ServerCodes::Data) { + break; + } + if (server_packet == ServerCodes::Progress) { + continue; + } + } +} + bool Client::Impl::ReceivePacket(uint64_t* server_packet) { uint64_t packet_type = 0; @@ -779,6 +822,14 @@ void Client::Insert(const std::string& table_name, const Block& block) { impl_->Insert(table_name, block); } +void Client::InsertQuery(const std::string& query, SelectCallback cb) { + impl_->InsertQuery(Query(query).OnData(cb)); +} + +void Client::InsertData(const Block& block) { + impl_->InsertData(block); +} + void Client::Ping() { impl_->Ping(); } diff --git a/lib/clickhouse-cpp/clickhouse/client.h b/lib/clickhouse-cpp/clickhouse/client.h index 6953e7d..5220fda 100644 --- a/lib/clickhouse-cpp/clickhouse/client.h +++ b/lib/clickhouse-cpp/clickhouse/client.h @@ -59,6 +59,11 @@ struct ClientOptions { DECLARE_FIELD(send_retries, int, SetSendRetries, 1); /// Amount of time to wait before next retry. DECLARE_FIELD(retry_timeout, std::chrono::seconds, SetRetryTimeout, std::chrono::seconds(5)); + /// Amount of time the socket waits for response. + /// If the timeout is set to zero (the default) then the operation will never timeout. + DECLARE_FIELD(socket_receive_timeout, std::chrono::seconds, SetSocketReceiveTimeout, std::chrono::seconds(0)); + /// Amount of time to wait for connection to be established + DECLARE_FIELD(socket_connect_timeout, std::chrono::seconds, SetSocketConnectTimeout, std::chrono::seconds(5)); /// Compression method. DECLARE_FIELD(compression_method, CompressionMethod, SetCompressionMethod, CompressionMethod::None); @@ -93,6 +98,10 @@ class Client { /// the data handler function \p cb. void SelectCancelable(const std::string& query, SelectCancelableCallback cb); + void InsertQuery(const std::string& query, SelectCallback cb); + + void InsertData(const Block& block); + /// Alias for Execute. void Select(const Query& query); diff --git a/typesToPhp.cpp b/typesToPhp.cpp index f6dcc90..61ec444 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -976,7 +976,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column auto tuple = columnRef->As(); if (fetch_mode & SC_FETCH_ONE) { array_init(arr); - for (size_t i = 0; i < tuple->tupleSize(); ++i) + for (size_t i = 0; i < tuple->TupleSize(); ++i) { convertToZval(arr, (*tuple)[i], row, "tuple", 1, 0); } @@ -984,7 +984,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column zval *return_tmp; SC_MAKE_STD_ZVAL(return_tmp); array_init(return_tmp); - for (size_t i = 0; i < tuple->tupleSize(); ++i) + for (size_t i = 0; i < tuple->TupleSize(); ++i) { convertToZval(return_tmp, (*tuple)[i], row, "tuple", 1, 0); } From a1a3ec8ef7960163463cafa582423a0fc2f87463 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Wed, 15 Apr 2020 14:59:57 -0400 Subject: [PATCH 10/16] Added support for ping() --- SeasClick.cpp | 37 +++++++++++++++++++++++++++++-------- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/SeasClick.cpp b/SeasClick.cpp index ec775ec..a9fde33 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -59,28 +59,32 @@ static PHP_METHOD(SEASCLICK_RES_NAME, __destruct); static PHP_METHOD(SEASCLICK_RES_NAME, select); static PHP_METHOD(SEASCLICK_RES_NAME, insert); static PHP_METHOD(SEASCLICK_RES_NAME, execute); +static PHP_METHOD(SEASCLICK_RES_NAME, ping); -ZEND_BEGIN_ARG_INFO_EX(SeasCilck_construct, 0, 0, 1) +ZEND_BEGIN_ARG_INFO_EX(SeasClick_construct, 0, 0, 1) ZEND_ARG_INFO(0, connectParams) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(SeasCilck_select, 0, 0, 3) +ZEND_BEGIN_ARG_INFO_EX(SeasClick_select, 0, 0, 3) ZEND_ARG_INFO(0, sql) ZEND_ARG_INFO(0, params) ZEND_ARG_INFO(0, fetch_mode) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(SeasCilck_insert, 0, 0, 3) +ZEND_BEGIN_ARG_INFO_EX(SeasClick_insert, 0, 0, 3) ZEND_ARG_INFO(0, table) ZEND_ARG_INFO(0, columns) ZEND_ARG_INFO(0, values) ZEND_END_ARG_INFO() -ZEND_BEGIN_ARG_INFO_EX(SeasCilck_execute, 0, 0, 2) +ZEND_BEGIN_ARG_INFO_EX(SeasClick_execute, 0, 0, 2) ZEND_ARG_INFO(0, sql) ZEND_ARG_INFO(0, params) ZEND_END_ARG_INFO() +ZEND_BEGIN_ARG_INFO_EX(SeasClick_ping, 0, 0, 0) +ZEND_END_ARG_INFO() + /* {{{ SeasClick_functions[] */ const zend_function_entry SeasClick_functions[] = { @@ -91,11 +95,12 @@ const zend_function_entry SeasClick_functions[] = const zend_function_entry SeasClick_methods[] = { - PHP_ME(SEASCLICK_RES_NAME, __construct, SeasCilck_construct, ZEND_ACC_PUBLIC | ZEND_ACC_CTOR) + PHP_ME(SEASCLICK_RES_NAME, __construct, SeasClick_construct, ZEND_ACC_PUBLIC | ZEND_ACC_CTOR) PHP_ME(SEASCLICK_RES_NAME, __destruct, NULL, ZEND_ACC_PUBLIC | ZEND_ACC_DTOR) - PHP_ME(SEASCLICK_RES_NAME, select, SeasCilck_select, ZEND_ACC_PUBLIC) - PHP_ME(SEASCLICK_RES_NAME, insert, SeasCilck_insert, ZEND_ACC_PUBLIC) - PHP_ME(SEASCLICK_RES_NAME, execute, SeasCilck_execute, ZEND_ACC_PUBLIC) + PHP_ME(SEASCLICK_RES_NAME, select, SeasClick_select, ZEND_ACC_PUBLIC) + PHP_ME(SEASCLICK_RES_NAME, insert, SeasClick_insert, ZEND_ACC_PUBLIC) + PHP_ME(SEASCLICK_RES_NAME, execute, SeasClick_execute, ZEND_ACC_PUBLIC) + PHP_ME(SEASCLICK_RES_NAME, ping, SeasClick_ping, ZEND_ACC_PUBLIC) PHP_FE_END }; @@ -327,6 +332,22 @@ void getInsertSql(string *sql, char *table_name, zval *columns) *sql = "INSERT INTO " + (string)table_name + " ( " + fields_section.str() + " ) VALUES"; } +/* {{{ proto bool ping() + */ +PHP_METHOD(SEASCLICK_RES_NAME, ping) +{ + int key = Z_OBJ_HANDLE(*getThis()); + Client *client = clientMap.at(key); + + try { + client->Ping(); + } catch (const std::exception& e) { + sc_zend_throw_exception(SeasClickException_ce, e.what(), 0 TSRMLS_CC); + } + + RETURN_TRUE; +} + /* {{{ proto array select(string sql, array params, int mode) */ PHP_METHOD(SEASCLICK_RES_NAME, select) From 857b842b7f37df34962841f30afeb79cb2901bc1 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Tue, 28 Apr 2020 14:47:52 -0400 Subject: [PATCH 11/16] Added support for composite arrays --- .travis.yml | 0 SeasClick.cpp | 5 +++++ tests/012.phpt | 0 tests/013.phpt | 0 tests/014.phpt | 0 tests/015.phpt | 0 travis/run-tests.sh | 0 typesToPhp.cpp | 33 +++++++++++++++++++++++++-------- 8 files changed, 30 insertions(+), 8 deletions(-) mode change 100644 => 100755 .travis.yml mode change 100644 => 100755 SeasClick.cpp mode change 100644 => 100755 tests/012.phpt mode change 100644 => 100755 tests/013.phpt mode change 100644 => 100755 tests/014.phpt mode change 100644 => 100755 tests/015.phpt mode change 100644 => 100755 travis/run-tests.sh diff --git a/.travis.yml b/.travis.yml old mode 100644 new mode 100755 diff --git a/SeasClick.cpp b/SeasClick.cpp old mode 100644 new mode 100755 index a9fde33..29a0dc3 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -510,6 +510,7 @@ PHP_METHOD(SEASCLICK_RES_NAME, insert) zval *return_tmp; for(size_t i = 0; i < columns_count; i++) { + zval *key = sc_zend_hash_index_find(columns_ht, i); SC_MAKE_STD_ZVAL(return_tmp); array_init(return_tmp); @@ -520,6 +521,10 @@ PHP_METHOD(SEASCLICK_RES_NAME, insert) throw std::runtime_error("The insert function needs to pass in a two-dimensional array"); } fzval = sc_zend_hash_index_find(Z_ARRVAL_P(pzval), i); + if (NULL == fzval && Z_TYPE_P(key) == IS_STRING) + { + fzval = sc_zend_hash_find(Z_ARRVAL_P(pzval), Z_STRVAL_P(key), Z_STRLEN_P(key)); + } if (NULL == fzval) { throw std::runtime_error("The number of parameters inserted per line is inconsistent"); diff --git a/tests/012.phpt b/tests/012.phpt old mode 100644 new mode 100755 diff --git a/tests/013.phpt b/tests/013.phpt old mode 100644 new mode 100755 diff --git a/tests/014.phpt b/tests/014.phpt old mode 100644 new mode 100755 diff --git a/tests/015.phpt b/tests/015.phpt old mode 100644 new mode 100755 diff --git a/travis/run-tests.sh b/travis/run-tests.sh old mode 100644 new mode 100755 diff --git a/typesToPhp.cpp b/typesToPhp.cpp index 61ec444..a80f0b8 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -42,6 +42,14 @@ extern "C" { using namespace clickhouse; using namespace std; +static std::time_t to_time_t(const std::string& str, bool is_date = true) +{ + std::tm t = {0}; + std::istringstream ss(str); + ss >> std::get_time(&t, is_date ? "%Y-%m-%d" : "%Y-%m-%d %H:%M:%S"); + return mktime(&t); +} + ColumnRef createColumn(TypeRef type) { switch (type->GetCode()) @@ -399,8 +407,12 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) SC_HASHTABLE_FOREACH_START2(values_ht, str_key, str_keylen, keytype, array_value) { - convert_to_long(array_value); - value->Append(Z_LVAL_P(array_value)); + if (Z_TYPE_P(array_value) == IS_STRING && memchr(Z_STRVAL_P(array_value), '-', Z_STRLEN_P(array_value)) != NULL) { + value->Append((long)to_time_t(Z_STRVAL_P(array_value), false)); + } else { + convert_to_long(array_value); + value->Append(Z_LVAL_P(array_value)); + } } SC_HASHTABLE_FOREACH_END(); @@ -413,8 +425,12 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) SC_HASHTABLE_FOREACH_START2(values_ht, str_key, str_keylen, keytype, array_value) { - convert_to_long(array_value); - value->Append(Z_LVAL_P(array_value)); + if (Z_TYPE_P(array_value) == IS_STRING && memchr(Z_STRVAL_P(array_value), '-', Z_STRLEN_P(array_value)) != NULL) { + value->Append((long)to_time_t(Z_STRVAL_P(array_value))); + } else { + convert_to_long(array_value); + value->Append(Z_LVAL_P(array_value)); + } } SC_HASHTABLE_FOREACH_END(); @@ -680,6 +696,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column break; } case Type::Code::UInt32: + case Type::Code::IPv4: { auto col = (*columnRef->As())[row]; if (is_array) @@ -692,7 +709,6 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } break; } - case Type::Code::Int8: { auto col = (*columnRef->As())[row]; @@ -745,7 +761,6 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } break; } - case Type::Code::UUID: { stringstream first; @@ -763,8 +778,9 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } break; } - case Type::Code::Float32: + case Type::Code::Decimal: + case Type::Code::Decimal32: { auto col = (*columnRef->As())[row]; stringstream stream; @@ -782,6 +798,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column break; } case Type::Code::Float64: + case Type::Code::Decimal64: { auto col = (*columnRef->As())[row]; if (is_array) @@ -794,7 +811,6 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column } break; } - case Type::Code::String: { auto col = (*columnRef->As())[row]; @@ -809,6 +825,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column break; } case Type::Code::FixedString: + case Type::Code::IPv6: { auto col = (*columnRef->As())[row]; if (is_array) From bb8756cf798763e82c15711aa238ee117cd2172b Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Fri, 8 May 2020 13:45:43 -0400 Subject: [PATCH 12/16] Added support for hex values for numbers --- .travis.yml | 0 SeasClick.cpp | 0 tests/012.phpt | 0 tests/013.phpt | 0 tests/014.phpt | 0 tests/015.phpt | 0 travis/run-tests.sh | 0 typesToPhp.cpp | 27 +++++++++++++++++++++++---- 8 files changed, 23 insertions(+), 4 deletions(-) mode change 100755 => 100644 .travis.yml mode change 100755 => 100644 SeasClick.cpp mode change 100755 => 100644 tests/012.phpt mode change 100755 => 100644 tests/013.phpt mode change 100755 => 100644 tests/014.phpt mode change 100755 => 100644 tests/015.phpt mode change 100755 => 100644 travis/run-tests.sh diff --git a/.travis.yml b/.travis.yml old mode 100755 new mode 100644 diff --git a/SeasClick.cpp b/SeasClick.cpp old mode 100755 new mode 100644 diff --git a/tests/012.phpt b/tests/012.phpt old mode 100755 new mode 100644 diff --git a/tests/013.phpt b/tests/013.phpt old mode 100755 new mode 100644 diff --git a/tests/014.phpt b/tests/014.phpt old mode 100755 new mode 100644 diff --git a/tests/015.phpt b/tests/015.phpt old mode 100755 new mode 100644 diff --git a/travis/run-tests.sh b/travis/run-tests.sh old mode 100755 new mode 100644 diff --git a/typesToPhp.cpp b/typesToPhp.cpp index a80f0b8..cccff83 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -201,8 +201,18 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) SC_HASHTABLE_FOREACH_START2(values_ht, str_key, str_keylen, keytype, array_value) { - convert_to_long(array_value); - value->Append(Z_LVAL_P(array_value)); + if ( + Z_TYPE_P(array_value) == IS_STRING && Z_STRLEN_P(array_value) >= 3 + && ( + *Z_STRVAL_P(array_value) == '0' && + ((*(Z_STRVAL_P(array_value) + 1) == 'x') || *(Z_STRVAL_P(array_value) + 1) == 'X') + ) + ) { + value->Append(strtoull(Z_STRVAL_P(array_value), NULL, 0)); + } else { + convert_to_long(array_value); + value->Append(Z_LVAL_P(array_value)); + } } SC_HASHTABLE_FOREACH_END(); @@ -243,8 +253,17 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) SC_HASHTABLE_FOREACH_START2(values_ht, str_key, str_keylen, keytype, array_value) { - convert_to_long(array_value); - value->Append(Z_LVAL_P(array_value)); + if ( + Z_TYPE_P(array_value) == IS_STRING && Z_STRLEN_P(array_value) >= 3 + && ( + *Z_STRVAL_P(array_value) == '0' && + ((*(Z_STRVAL_P(array_value) + 1) == 'x') || *(Z_STRVAL_P(array_value) + 1) == 'X') + )) { + value->Append(strtoul(Z_STRVAL_P(array_value), NULL, 0)); + } else { + convert_to_long(array_value); + value->Append(Z_LVAL_P(array_value)); + } } SC_HASHTABLE_FOREACH_END(); From 88e10f48db2a07de6f95eb5be7459ef490f108f8 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Fri, 15 May 2020 23:01:19 -0400 Subject: [PATCH 13/16] Added timezone conversion work-around --- typesToPhp.cpp | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/typesToPhp.cpp b/typesToPhp.cpp index cccff83..838b8e8 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -447,8 +447,12 @@ ColumnRef insertColumn(TypeRef type, zval *value_zval) if (Z_TYPE_P(array_value) == IS_STRING && memchr(Z_STRVAL_P(array_value), '-', Z_STRLEN_P(array_value)) != NULL) { value->Append((long)to_time_t(Z_STRVAL_P(array_value))); } else { + time_t t; + struct tm *tz; convert_to_long(array_value); - value->Append(Z_LVAL_P(array_value)); + t = (time_t) Z_LVAL_P(array_value); + tz = localtime(&t); + value->Append(Z_LVAL_P(array_value) + tz->tm_gmtoff); } } SC_HASHTABLE_FOREACH_END(); From 67a42c4605f9d445476fd3d7d47ce17ff7d7eac3 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Tue, 26 May 2020 15:17:38 -0400 Subject: [PATCH 14/16] date fixes --- typesToPhp.cpp | 40 ++++++++++++++++++++++++++++++++-------- 1 file changed, 32 insertions(+), 8 deletions(-) diff --git a/typesToPhp.cpp b/typesToPhp.cpp index 838b8e8..16e1e36 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -871,8 +871,12 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column char buffer[32]; size_t l; std::time_t t = (long)col->As()->At(row); - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); - sc_add_next_index_stringl(arr, buffer, l, 1); + if (t > 0) { + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); + sc_add_next_index_stringl(arr, buffer, l, 1); + } else { + add_next_index_null(arr); + } } else { add_next_index_long(arr, (long)col->As()->At(row)); } @@ -883,8 +887,16 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column char buffer[32]; size_t l; std::time_t t = (long)col->As()->At(row); - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); - SC_SINGLE_STRING(buffer, l); + if (t > 0) { + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); + SC_SINGLE_STRING(buffer, l); + } else { + if (fetch_mode & SC_FETCH_ONE) { + ZVAL_NULL(arr); + } else { + sc_add_assoc_null_ex(arr, column_name.c_str(), column_name.length()); + } + } } else { if (fetch_mode & SC_FETCH_ONE) { ZVAL_LONG(arr, (long)col->As()->At(row)); @@ -904,8 +916,12 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column char buffer[16]; size_t l; std::time_t t = (long)col->As()->At(row); - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); - sc_add_next_index_stringl(arr, buffer, l, 1); + if (t > 0) { + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); + sc_add_next_index_stringl(arr, buffer, l, 1); + } else { + add_next_index_null(arr); + } } else { add_next_index_long(arr, (long)col->As()->At(row)); } @@ -916,8 +932,16 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column char buffer[16]; size_t l; std::time_t t = (long)col->As()->At(row); - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); - SC_SINGLE_STRING(buffer, l); + if (t > 0) { + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); + SC_SINGLE_STRING(buffer, l); + } else { + if (fetch_mode & SC_FETCH_ONE) { + ZVAL_NULL(arr); + } else { + sc_add_assoc_null_ex(arr, column_name.c_str(), column_name.length()); + } + } } else { if (fetch_mode & SC_FETCH_ONE) { ZVAL_LONG(arr, (long)col->As()->At(row)); From ccbdb1d2611e17930bc9d69af67167c1fd030471 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Tue, 7 Jul 2020 09:27:39 -0400 Subject: [PATCH 15/16] date timezone fix --- typesToPhp.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/typesToPhp.cpp b/typesToPhp.cpp index 16e1e36..aa2db35 100644 --- a/typesToPhp.cpp +++ b/typesToPhp.cpp @@ -872,7 +872,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column size_t l; std::time_t t = (long)col->As()->At(row); if (t > 0) { - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", gmtime(&t)); sc_add_next_index_stringl(arr, buffer, l, 1); } else { add_next_index_null(arr); @@ -888,7 +888,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column size_t l; std::time_t t = (long)col->As()->At(row); if (t > 0) { - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", localtime(&t)); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d %H:%M:%S", gmtime(&t)); SC_SINGLE_STRING(buffer, l); } else { if (fetch_mode & SC_FETCH_ONE) { @@ -917,7 +917,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column size_t l; std::time_t t = (long)col->As()->At(row); if (t > 0) { - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", gmtime(&t)); sc_add_next_index_stringl(arr, buffer, l, 1); } else { add_next_index_null(arr); @@ -933,7 +933,7 @@ void convertToZval(zval *arr, const ColumnRef& columnRef, int row, string column size_t l; std::time_t t = (long)col->As()->At(row); if (t > 0) { - l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", localtime(&t)); + l = strftime(buffer, sizeof(buffer), "%Y-%m-%d", gmtime(&t)); SC_SINGLE_STRING(buffer, l); } else { if (fetch_mode & SC_FETCH_ONE) { From 3d021050ad08d1dd3dad32bf53066568a55c5572 Mon Sep 17 00:00:00 2001 From: Ilia Alshanetsky Date: Thu, 14 Oct 2021 15:06:10 -0400 Subject: [PATCH 16/16] Fixed 7.4 compilation --- SeasClick.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/SeasClick.cpp b/SeasClick.cpp index 29a0dc3..4f504de 100644 --- a/SeasClick.cpp +++ b/SeasClick.cpp @@ -139,7 +139,7 @@ PHP_MINIT_FUNCTION(SeasClick) REGISTER_SC_CLASS_CONST_LONG("DATE_AS_STRINGS", (zend_long)SC_FETCH_DATE_AS_STRINGS); REGISTER_SC_CLASS_CONST_LONG("FETCH_COLUMN", (zend_long)SC_FETCH_COLUMN); - SeasClick_ce->ce_flags = ZEND_ACC_IMPLICIT_PUBLIC; + SeasClick_ce->ce_flags |= ZEND_ACC_FINAL; return SUCCESS; } /* }}} */