HappenLee commented on code in PR #63309:
URL: https://github.com/apache/doris/pull/63309#discussion_r3361046233
##########
be/src/util/jsonb_parser_simd.h:
##########
@@ -281,54 +293,151 @@ struct JsonbParser {
case simdjson::ondemand::number_type::signed_integer:
case simdjson::ondemand::number_type::unsigned_integer: {
int128_t val = num.is_int64() ? (int128_t)num.get_int64() :
(int128_t)num.get_uint64();
- bool success = false;
- if (val >= std::numeric_limits<int8_t>::min() &&
- val <= std::numeric_limits<int8_t>::max()) {
- success = writer.writeInt8((int8_t)val);
- } else if (val >= std::numeric_limits<int16_t>::min() &&
- val <= std::numeric_limits<int16_t>::max()) {
- success = writer.writeInt16((int16_t)val);
- } else if (val >= std::numeric_limits<int32_t>::min() &&
- val <= std::numeric_limits<int32_t>::max()) {
- success = writer.writeInt32((int32_t)val);
- } else if (val >= std::numeric_limits<int64_t>::min() &&
- val <= std::numeric_limits<int64_t>::max()) {
- success = writer.writeInt64((int64_t)val);
- } else { // INT128
- success = writer.writeInt128(val);
+ RETURN_IF_ERROR(write_int128(val, writer));
+ break;
+ }
+ case simdjson::ondemand::number_type::big_integer: {
+ RETURN_IF_ERROR(write_number_from_raw_json(raw_string, writer));
+ break;
+ }
+ }
+ return Status::OK();
+ }
+
+ static bool is_json_number_space(char c) {
+ return c == ' ' || c == '\t' || c == '\n' || c == '\r';
+ }
+
+ static std::string_view trim_json_number(std::string_view raw_number) {
+ while (!raw_number.empty() &&
is_json_number_space(raw_number.front())) {
+ raw_number.remove_prefix(1);
+ }
+ while (!raw_number.empty() && is_json_number_space(raw_number.back()))
{
+ raw_number.remove_suffix(1);
+ }
+ return raw_number;
+ }
+
+ static bool is_json_number_digit(char c) { return c >= '0' && c <= '9'; }
+
+ static Status validate_json_number(std::string_view raw_number, bool&
is_integer) {
+ if (raw_number.empty()) {
+ return Status::InvalidArgument("empty number");
+ }
+
+ size_t pos = 0;
+ if (raw_number[pos] == '-') {
+ ++pos;
+ if (pos == raw_number.size()) {
+ return Status::InvalidArgument("invalid number, raw string is:
" +
+ std::string(raw_number));
}
+ }
- if (!success) {
- return Status::InvalidArgument("writeInt failed");
+ if (raw_number[pos] == '0') {
+ ++pos;
+ } else if (raw_number[pos] >= '1' && raw_number[pos] <= '9') {
+ while (pos < raw_number.size() &&
is_json_number_digit(raw_number[pos])) {
+ ++pos;
}
- break;
+ } else {
+ return Status::InvalidArgument("invalid number, raw string is: " +
+ std::string(raw_number));
}
- case simdjson::ondemand::number_type::big_integer: {
- StringParser::ParseResult result;
- auto val =
StringParser::string_to_int<int128_t>(raw_string.data(), raw_string.size(),
+
+ bool has_fraction = false;
+ if (pos < raw_number.size() && raw_number[pos] == '.') {
+ has_fraction = true;
+ ++pos;
+ if (pos == raw_number.size() ||
!is_json_number_digit(raw_number[pos])) {
+ return Status::InvalidArgument("invalid number, raw string is:
" +
+ std::string(raw_number));
+ }
+ while (pos < raw_number.size() &&
is_json_number_digit(raw_number[pos])) {
+ ++pos;
+ }
+ }
+
+ bool has_exponent = false;
+ if (pos < raw_number.size() && (raw_number[pos] == 'e' ||
raw_number[pos] == 'E')) {
+ has_exponent = true;
+ ++pos;
+ if (pos < raw_number.size() && (raw_number[pos] == '+' ||
raw_number[pos] == '-')) {
+ ++pos;
+ }
+ if (pos == raw_number.size() ||
!is_json_number_digit(raw_number[pos])) {
+ return Status::InvalidArgument("invalid number, raw string is:
" +
+ std::string(raw_number));
+ }
+ while (pos < raw_number.size() &&
is_json_number_digit(raw_number[pos])) {
+ ++pos;
+ }
+ }
+
+ if (pos != raw_number.size()) {
+ return Status::InvalidArgument("simdjson parse exception: trailing
content");
+ }
+ is_integer = !has_fraction && !has_exponent;
+ return Status::OK();
+ }
+
+ static Status write_int128(int128_t val, JsonbWriter& writer) {
+ bool success = false;
+ if (val >= std::numeric_limits<int8_t>::min() &&
+ val <= std::numeric_limits<int8_t>::max()) {
+ success = writer.writeInt8((int8_t)val);
+ } else if (val >= std::numeric_limits<int16_t>::min() &&
+ val <= std::numeric_limits<int16_t>::max()) {
+ success = writer.writeInt16((int16_t)val);
+ } else if (val >= std::numeric_limits<int32_t>::min() &&
+ val <= std::numeric_limits<int32_t>::max()) {
+ success = writer.writeInt32((int32_t)val);
+ } else if (val >= std::numeric_limits<int64_t>::min() &&
+ val <= std::numeric_limits<int64_t>::max()) {
+ success = writer.writeInt64((int64_t)val);
+ } else { // INT128
+ success = writer.writeInt128(val);
+ }
+
+ if (!success) {
+ return Status::InvalidArgument("writeInt failed");
+ }
+ return Status::OK();
+ }
+
+ static Status write_number_from_raw_json(const char* pch, size_t len,
JsonbWriter& writer) {
Review Comment:
没必要实现这个重载，外面直接直接用户的时候调用stringview没什么代价
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
