Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_bitmap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
#include "core/string_buffer.hpp"
#include "core/string_ref.h"
#include "core/value/bitmap_value.h"
#include "util/io_helper.h"

namespace doris {
#include "common/compile_check_begin.h"
Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_date.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@
#include "core/types.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp"
#include "util/io_helper.h"

namespace doris {
bool DataTypeDate::equals(const IDataType& rhs) const {
Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_date_or_datetime_v2.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,6 @@
#include "exprs/function/cast/cast_to_datetimev2_impl.hpp"
#include "exprs/function/cast/cast_to_datev2_impl.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

namespace doris {
#include "common/compile_check_begin.h"
Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_date_time.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

namespace doris {

Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_decimal.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
#include "core/value/decimalv2_value.h"
#include "exec/common/int_exp.h"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"
#include "util/string_parser.hpp"

namespace doris {
Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_hll.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
#include "core/column/column_complex.h"
#include "core/column/column_const.h"
#include "core/string_buffer.hpp"
#include "util/io_helper.h"
#include "util/slice.h"

namespace doris {
Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_ipv4.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,6 @@
#include "core/data_type/data_type.h"
#include "core/string_buffer.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

namespace doris {

Expand Down
1 change: 0 additions & 1 deletion be/src/core/data_type/data_type_ipv6.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
#include "core/string_buffer.hpp"
#include "core/value/ipv6_value.h"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

namespace doris {
bool DataTypeIPv6::equals(const IDataType& rhs) const {
Expand Down
7 changes: 5 additions & 2 deletions be/src/core/data_type/data_type_number_base.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,6 @@
#include "core/types.h"
#include "core/value/large_int_value.h"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"
#include "util/mysql_global.h"
#include "util/string_parser.hpp"
#include "util/to_string.h"
Expand All @@ -55,7 +54,11 @@ std::string DataTypeNumberBase<T>::to_string(
if constexpr (std::is_same<typename PrimitiveTypeTraits<T>::CppType, int128_t>::value ||
std::is_same<typename PrimitiveTypeTraits<T>::CppType, uint128_t>::value ||
std::is_same<typename PrimitiveTypeTraits<T>::CppType, UInt128>::value) {
return int128_to_string(value);
if constexpr (std::is_same<typename PrimitiveTypeTraits<T>::CppType, int128_t>::value) {
return CastToString::from_int128(value);
} else {
return CastToString::from_uint128(value);
}
} else if constexpr (std::is_integral<typename PrimitiveTypeTraits<T>::CppType>::value) {
return std::to_string(value);
} else if constexpr (T == TYPE_DATETIME || T == TYPE_DATE) {
Expand Down
3 changes: 1 addition & 2 deletions be/src/core/data_type/data_type_quantilestate.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
#include "core/column/column_complex.h"
#include "core/string_buffer.hpp"
#include "core/string_ref.h"
#include "util/io_helper.h"
#include "util/slice.h"

namespace doris {
Expand Down Expand Up @@ -176,4 +175,4 @@ void DataTypeQuantileState::deserialize_as_stream(QuantileState& value, BufferRe
value.deserialize(ref.to_slice());
}

} // namespace doris
} // namespace doris
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,6 @@
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_base.h"
#include "exprs/function/cast/cast_to_date_or_datetime_impl.hpp"
#include "util/io_helper.h"

namespace doris {
#include "common/compile_check_begin.h"
Expand Down Expand Up @@ -77,9 +76,13 @@ Status DataTypeDateSerDe<T>::deserialize_one_cell_from_json(
slice.trim_quote();
}
VecDateTimeValue val;
if (StringRef str(slice.data, slice.size); !read_date_text_impl(val, str)) {
StringRef str(slice.data, slice.size);
CastParameters params;
if (!CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE>(
str, val, nullptr, params)) {
return Status::InvalidArgument("parse date fail, string: '{}'", str.to_string());
}
val.cast_to_date();
column_data.insert_value(val);
return Status::OK();
}
Expand Down Expand Up @@ -124,9 +127,13 @@ Status DataTypeDateTimeSerDe::deserialize_one_cell_from_json(IColumn& column, Sl
slice.trim_quote();
}
VecDateTimeValue val;
if (StringRef str(slice.data, slice.size); !read_datetime_text_impl(val, str)) {
StringRef str(slice.data, slice.size);
CastParameters params;
if (!CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE_TIME>(
str, val, nullptr, params)) {
return Status::InvalidArgument("parse datetime fail, string: '{}'", str.to_string());
}
val.to_datetime();
column_data.insert_value(val);
return Status::OK();
}
Expand Down Expand Up @@ -225,10 +232,10 @@ Status DataTypeDateSerDe<T>::_read_column_from_arrow(IColumn& column,
} else if (arrow_array->type()->id() == arrow::Type::STRING) {
// to be compatible with old version, we use string type for date.
const auto* concrete_array = dynamic_cast<const arrow::StringArray*>(arrow_array);
CastParameters params;
for (auto value_i = start; value_i < end; ++value_i) {
auto val_str = concrete_array->GetString(value_i);
VecDateTimeValue v;
CastParameters params;
CastToDateOrDatetime::from_string_non_strict_mode<DatelikeTargetType::DATE_TIME>(
{val_str.c_str(), val_str.length()}, v, &ctz, params);
if constexpr (is_date) {
Expand Down
5 changes: 3 additions & 2 deletions be/src/core/data_type_serde/data_type_datetimev2_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,6 @@
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_datetimev2_impl.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

enum {
DIVISOR_FOR_SECOND = 1,
Expand Down Expand Up @@ -346,7 +345,9 @@ Status DataTypeDateTimeV2SerDe::deserialize_one_cell_from_json(IColumn& column,
slice.trim_quote();
}
DateV2Value<DateTimeV2ValueType> val;
if (StringRef str(slice.data, slice.size); !read_datetime_v2_text_impl(val, str, _scale)) {
StringRef str(slice.data, slice.size);
CastParameters params;
if (!CastToDatetimeV2::from_string_non_strict_mode(str, val, nullptr, _scale, params)) {
return Status::InvalidArgument("parse date fail, string: '{}'", str.to_string());
}
column_data.insert_value(val);
Expand Down
5 changes: 3 additions & 2 deletions be/src/core/data_type_serde/data_type_datev2_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_datev2_impl.hpp"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

namespace doris {

Expand Down Expand Up @@ -81,7 +80,9 @@ Status DataTypeDateV2SerDe::deserialize_one_cell_from_json(IColumn& column, Slic
}
auto& column_data = assert_cast<ColumnDateV2&>(column);
DateV2Value<DateV2ValueType> val;
if (StringRef str(slice.data, slice.size); !read_date_v2_text_impl(val, str)) {
StringRef str(slice.data, slice.size);
CastParameters params;
if (!CastToDateV2::from_string_non_strict_mode(str, val, nullptr, params)) {
return Status::InvalidArgument("parse date fail, string: '{}'", str.to_string());
}
column_data.insert_value(val);
Expand Down
18 changes: 13 additions & 5 deletions be/src/core/data_type_serde/data_type_decimal_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <type_traits>

#include "arrow/type.h"
#include "common/cast_set.h"
#include "common/consts.h"
#include "core/column/column.h"
#include "core/column/column_decimal.h"
Expand All @@ -36,10 +37,10 @@
#include "exprs/function/cast/cast_to_string.h"
#include "orc/Int128.hh"
#include "storage/tablet/tablet_schema.h"
#include "util/io_helper.h"
#include "util/jsonb_document.h"
#include "util/jsonb_document_cast.h"
#include "util/jsonb_writer.h"
#include "util/string_parser.hpp"

namespace doris {
// #include "common/compile_check_begin.h"
Expand Down Expand Up @@ -140,7 +141,7 @@ Status DataTypeDecimalSerDe<T>::from_olap_string(const std::string& str, Field&
// DecimalV2: zonemap stores "integer.fraction" with 9 zero-padded fractional digits.
// E.g., DecimalV2 value 123.456 → to_olap_string() → "123.456000000".
// Caller sets ignore_scale=false → parse with scale=9 → correctly restores the value.
// Note: read_decimal_text_impl() currently hardcodes DecimalV2Value::SCALE=9 for
// Note: CastToDecimal::from_string() currently hardcodes DecimalV2Value::SCALE=9 for
// DecimalV2, so the passed-in scale is effectively ignored. But callers should still
// set ignore_scale=false for semantic correctness.
if (!CastToDecimal::from_string(StringRef(str), to, static_cast<UInt32>(precision),
Expand Down Expand Up @@ -207,9 +208,16 @@ Status DataTypeDecimalSerDe<T>::deserialize_one_cell_from_json(IColumn& column,
auto& column_data = assert_cast<ColumnDecimal<T>&>(column).get_data();
FieldType val = {};
StringRef str_ref(slice.data, slice.size);
StringParser::ParseResult res =
read_decimal_text_impl<get_primitive_type(), FieldType>(val, str_ref, precision, scale);
if (res == StringParser::PARSE_SUCCESS || res == StringParser::PARSE_UNDERFLOW) {
StringParser::ParseResult result = StringParser::PARSE_SUCCESS;
if constexpr (T == TYPE_DECIMALV2) {
val = DecimalV2Value(StringParser::string_to_decimal<TYPE_DECIMALV2>(
str_ref.data, cast_set<Int32>(str_ref.size), DecimalV2Value::PRECISION,
DecimalV2Value::SCALE, &result));
} else {
val.value = StringParser::string_to_decimal<T>(str_ref.data, cast_set<Int32>(str_ref.size),
precision, scale, &result);
Comment on lines +211 to +218
}
if (result == StringParser::PARSE_SUCCESS || result == StringParser::PARSE_UNDERFLOW) {
column_data.emplace_back(val);
return Status::OK();
}
Expand Down
4 changes: 2 additions & 2 deletions be/src/core/data_type_serde/data_type_ipv4_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,6 @@
#include "core/types.h"
#include "exprs/function/cast/cast_to_ip.h"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"

namespace doris {
#include "common/compile_check_begin.h"
Expand Down Expand Up @@ -68,7 +67,8 @@ Status DataTypeIPv4SerDe::deserialize_one_cell_from_json(IColumn& column, Slice&
auto& column_data = reinterpret_cast<ColumnIPv4&>(column);
StringRef str(slice.data, slice.size);
IPv4 val = 0;
if (!read_ipv4_text_impl(val, str)) {
CastParameters params;
if (!CastToIPv4::from_string(str, val, params)) {
return Status::InvalidArgument("parse ipv4 fail, string: '{}'", str.to_string());
}
column_data.insert_value(val);
Expand Down
4 changes: 2 additions & 2 deletions be/src/core/data_type_serde/data_type_ipv6_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@
#include "core/types.h"
#include "exprs/function/cast/cast_to_ip.h"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"
#include "util/jsonb_writer.h"

namespace doris {
Expand Down Expand Up @@ -91,7 +90,8 @@ Status DataTypeIPv6SerDe::deserialize_one_cell_from_json(IColumn& column, Slice&
auto& column_data = reinterpret_cast<ColumnIPv6&>(column);
StringRef str(slice.data, slice.size);
IPv6 val = 0;
if (!read_ipv6_text_impl(val, str)) {
CastParameters params;
if (!CastToIPv6::from_string(str, val, params)) {
return Status::InvalidArgument("parse ipv6 fail, string: '{}'", str.to_string());
}
column_data.insert_value(val);
Expand Down
23 changes: 13 additions & 10 deletions be/src/core/data_type_serde/data_type_number_serde.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,6 @@
#include "exprs/function/cast/cast_to_string.h"
#include "storage/olap_common.h"
#include "storage/types.h"
#include "util/io_helper.h"
#include "util/jsonb_document.h"
#include "util/jsonb_document_cast.h"
#include "util/jsonb_writer.h"
Expand Down Expand Up @@ -167,20 +166,23 @@ Status DataTypeNumberSerDe<T>::deserialize_one_cell_from_json(IColumn& column, S
return Status::InvalidArgument("uint128 is not support");
} else if constexpr (is_float_or_double(T) || T == TYPE_TIMEV2 || T == TYPE_TIME) {
typename PrimitiveTypeTraits<T>::CppType val = 0;
if (!try_read_float_text(val, str_ref)) {
CastParameters params;
if (!CastToFloat::from_string(str_ref, val, params)) {
return Status::InvalidArgument("parse number fail, string: '{}'", slice.to_string());
}
column_data.insert_value(val);
} else if constexpr (T == TYPE_BOOLEAN) {
// Note: here we should handle the bool type
typename PrimitiveTypeTraits<T>::CppType val = 0;
if (!try_read_bool_text(val, str_ref)) {
CastParameters params;
if (!CastToBool::from_string(str_ref, val, params)) {
return Status::InvalidArgument("parse boolean fail, string: '{}'", slice.to_string());
}
column_data.insert_value(val);
} else if constexpr (is_int_or_bool(T)) {
typename PrimitiveTypeTraits<T>::CppType val = 0;
if (!try_read_int_text(val, str_ref)) {
CastParameters params;
if (!CastToInt::from_string<false>(str_ref, val, params)) {
return Status::InvalidArgument("parse number fail, string: '{}'", slice.to_string());
}
column_data.insert_value(val);
Expand All @@ -206,7 +208,7 @@ Status DataTypeNumberSerDe<T>::serialize_one_cell_to_json(const IColumn& column,
row_num = result.second;
auto data = assert_cast<const ColumnType&>(*ptr).get_element(row_num);
if constexpr (T == TYPE_IPV6) {
std::string hex = int128_to_string(data);
std::string hex = CastToString::from_uint128(data);
bw.write(hex.data(), hex.size());
} else if constexpr (T == TYPE_FLOAT || T == TYPE_DOUBLE) {
auto str = CastToString::from_number(data);
Expand Down Expand Up @@ -247,6 +249,7 @@ Status DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
if (arrow_array->type_id() == arrow::Type::STRING) {
const auto* concrete_array = dynamic_cast<const arrow::StringArray*>(arrow_array);
std::shared_ptr<arrow::Buffer> buffer = concrete_array->value_data();
CastParameters params;

const auto* offsets_data = concrete_array->value_offsets()->data();
const size_t offset_size = sizeof(int32_t);
Expand All @@ -267,7 +270,7 @@ Status DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
if constexpr (T == TYPE_DATETIMEV2 || T == TYPE_TIMESTAMPTZ) {
StringRef str_ref(raw_data, raw_data_len);
UInt64 val = 0;
if (!try_read_int_text(val, str_ref)) {
if (!CastToInt::from_string<false>(str_ref, val, params)) {
return Status::Error(ErrorCode::INVALID_ARGUMENT,
"parse number fail, string: '{}'",
std::string(str_ref.data, str_ref.size).c_str());
Expand All @@ -277,7 +280,7 @@ Status DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
} else if constexpr (T == TYPE_DATE || T == TYPE_DATETIME) {
StringRef str_ref(raw_data, raw_data_len);
Int64 val = 0;
if (!try_read_int_text(val, str_ref)) {
if (!CastToInt::from_string<false>(str_ref, val, params)) {
return Status::Error(ErrorCode::INVALID_ARGUMENT,
"parse number fail, string: '{}'",
std::string(str_ref.data, str_ref.size).c_str());
Expand All @@ -287,7 +290,7 @@ Status DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
} else if constexpr (T == TYPE_DATEV2) {
StringRef str_ref(raw_data, raw_data_len);
UInt32 val = 0;
if (!try_read_int_text(val, str_ref)) {
if (!CastToInt::from_string<false>(str_ref, val, params)) {
return Status::Error(ErrorCode::INVALID_ARGUMENT,
"parse number fail, string: '{}'",
std::string(str_ref.data, str_ref.size).c_str());
Expand All @@ -297,7 +300,7 @@ Status DataTypeNumberSerDe<T>::read_column_from_arrow(IColumn& column,
} else {
Int128 val = 0;
StringRef str_ref(raw_data, raw_data_len);
if (!try_read_int_text(val, str_ref)) {
if (!CastToInt::from_string<false>(str_ref, val, params)) {
return Status::Error(ErrorCode::INVALID_ARGUMENT,
"parse number fail, string: '{}'",
std::string(str_ref.data, str_ref.size).c_str());
Expand Down Expand Up @@ -717,7 +720,7 @@ template <PrimitiveType PT, bool is_strict_mode>
bool try_parse_impl(typename PrimitiveTypeTraits<PT>::CppType& x, const StringRef& str_ref,
CastParameters& params) {
if constexpr (is_float_or_double(PT)) {
return try_read_float_text(x, str_ref);
return CastToFloat::from_string(str_ref, x, params);
} else if constexpr (PT == TYPE_BOOLEAN) {
return CastToBool::from_string(str_ref, x, params);
} else if constexpr (is_int(PT)) {
Expand Down
2 changes: 1 addition & 1 deletion be/src/core/data_type_serde/data_type_serde.h
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ class DataTypeSerDe {
* E.g., DecimalV2 value 123.456 is stored as "123.456000000";
* parsing with scale=9 correctly restores the original value.
*
* Note: for DecimalV2, read_decimal_text_impl() currently hardcodes
* Note: for DecimalV2, CastToDecimal::from_string() currently hardcodes
* DecimalV2Value::SCALE=9 regardless of the passed-in scale, so the flag
* does not actually affect DecimalV2 parsing today. However, callers should
* still set it correctly for semantic clarity and future-proofing.
Expand Down
1 change: 0 additions & 1 deletion be/src/core/field.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
#include "core/value/timestamptz_value.h"
#include "core/value/vdatetime_value.h"
#include "exprs/function/cast/cast_to_string.h"
#include "util/io_helper.h"
#include "util/var_int.h"

namespace doris {
Expand Down
1 change: 0 additions & 1 deletion be/src/exec/common/hash_table/hash_table.h
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@
#include "common/status.h"
#include "core/types.h"
#include "runtime/runtime_profile.h"
#include "util/io_helper.h"

/** NOTE HashTable could only be used for memmoveable (position independent) types.
* Example: std::string is not position independent in libstdc++ with C++11 ABI or in libc++.
Expand Down
Loading
Loading