This is an automated email from the ASF dual-hosted git repository.
wjones127 pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new b7fa29bc6f GH-34552: [C++][Parquet] Sync parquet.thrift from upstream
(#34553)
b7fa29bc6f is described below
commit b7fa29bc6f934848ce4946f408cbc8fa98ae784b
Author: Gang Wu <[email protected]>
AuthorDate: Wed Mar 15 03:27:12 2023 +0800
GH-34552: [C++][Parquet] Sync parquet.thrift from upstream (#34553)
### Rationale for this change
The parquet.thrift file has been updated with various comments to minimize
confusion or misuse about crc, page padding, and NaN handling. These are
essential guidelines for recent changes in the parquet-cpp. Therefore it is a
good time to sync this file.
In addition, the generated sources were built with thrift 0.13. We should
rebuild them with latest thrift tool to embrace modern cpp.
### What changes are included in this PR?
- Update `parquet.thrift` file from latest `apache/parquet-format` repo.
- Build it with thrift 0.17.
### Are these changes tested?
Make sure all tests pass.
### Are there any user-facing changes?
No.
* Closes: #34552
Authored-by: Gang Wu <[email protected]>
Signed-off-by: Will Jones <[email protected]>
---
cpp/src/generated/parquet_types.cpp | 520 +++++++++++---
cpp/src/generated/parquet_types.h | 1351 +++++++++++++++++++++++++++++------
cpp/src/parquet/parquet.thrift | 116 +--
3 files changed, 1612 insertions(+), 375 deletions(-)
diff --git a/cpp/src/generated/parquet_types.cpp
b/cpp/src/generated/parquet_types.cpp
index cccd92e2ec..b19f2eb19f 100644
--- a/cpp/src/generated/parquet_types.cpp
+++ b/cpp/src/generated/parquet_types.cpp
@@ -1,5 +1,5 @@
/**
- * Autogenerated by Thrift Compiler (0.13.0)
+ * Autogenerated by Thrift Compiler (0.17.0)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
@@ -33,7 +33,7 @@ const char* _kTypeNames[] = {
"BYTE_ARRAY",
"FIXED_LEN_BYTE_ARRAY"
};
-const std::map<int, const char*>
_Type_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(8, _kTypeValues,
_kTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_Type_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(8, _kTypeValues,
_kTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const Type::type& val) {
std::map<int, const char*>::const_iterator it =
_Type_VALUES_TO_NAMES.find(val);
@@ -55,54 +55,254 @@ std::string to_string(const Type::type& val) {
}
int _kConvertedTypeValues[] = {
+ /**
+ * a BYTE_ARRAY actually contains UTF8 encoded chars
+ */
ConvertedType::UTF8,
+ /**
+ * a map is converted as an optional field containing a repeated key/value
pair
+ */
ConvertedType::MAP,
+ /**
+ * a key/value pair is converted into a group of two fields
+ */
ConvertedType::MAP_KEY_VALUE,
+ /**
+ * a list is converted into an optional field containing a repeated field
for its
+ * values
+ */
ConvertedType::LIST,
+ /**
+ * an enum is converted into a binary field
+ */
ConvertedType::ENUM,
+ /**
+ * A decimal value.
+ *
+ * This may be used to annotate binary or fixed primitive types. The
+ * underlying byte array stores the unscaled value encoded as two's
+ * complement using big-endian byte order (the most significant byte is the
+ * zeroth element). The value of the decimal is the value * 10^{-scale}.
+ *
+ * This must be accompanied by a (maximum) precision and a scale in the
+ * SchemaElement. The precision specifies the number of digits in the decimal
+ * and the scale stores the location of the decimal point. For example 1.23
+ * would have precision 3 (3 total digits) and scale 2 (the decimal point is
+ * 2 digits over).
+ */
ConvertedType::DECIMAL,
+ /**
+ * A Date
+ *
+ * Stored as days since Unix epoch, encoded as the INT32 physical type.
+ *
+ */
ConvertedType::DATE,
+ /**
+ * A time
+ *
+ * The total number of milliseconds since midnight. The value is stored
+ * as an INT32 physical type.
+ */
ConvertedType::TIME_MILLIS,
+ /**
+ * A time.
+ *
+ * The total number of microseconds since midnight. The value is stored as
+ * an INT64 physical type.
+ */
ConvertedType::TIME_MICROS,
+ /**
+ * A date/time combination
+ *
+ * Date and time recorded as milliseconds since the Unix epoch. Recorded as
+ * a physical type of INT64.
+ */
ConvertedType::TIMESTAMP_MILLIS,
+ /**
+ * A date/time combination
+ *
+ * Date and time recorded as microseconds since the Unix epoch. The value is
+ * stored as an INT64 physical type.
+ */
ConvertedType::TIMESTAMP_MICROS,
+ /**
+ * An unsigned integer value.
+ *
+ * The number describes the maximum number of meaningful data bits in
+ * the stored value. 8, 16 and 32 bit values are stored using the
+ * INT32 physical type. 64 bit values are stored using the INT64
+ * physical type.
+ *
+ */
ConvertedType::UINT_8,
ConvertedType::UINT_16,
ConvertedType::UINT_32,
ConvertedType::UINT_64,
+ /**
+ * A signed integer value.
+ *
+ * The number describes the maximum number of meaningful data bits in
+ * the stored value. 8, 16 and 32 bit values are stored using the
+ * INT32 physical type. 64 bit values are stored using the INT64
+ * physical type.
+ *
+ */
ConvertedType::INT_8,
ConvertedType::INT_16,
ConvertedType::INT_32,
ConvertedType::INT_64,
+ /**
+ * An embedded JSON document
+ *
+ * A JSON document embedded within a single UTF8 column.
+ */
ConvertedType::JSON,
+ /**
+ * An embedded BSON document
+ *
+ * A BSON document embedded within a single BINARY column.
+ */
ConvertedType::BSON,
+ /**
+ * An interval of time
+ *
+ * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12
+ * This data is composed of three separate little endian unsigned
+ * integers. Each stores a component of a duration of time. The first
+ * integer identifies the number of months associated with the duration,
+ * the second identifies the number of days associated with the duration
+ * and the third identifies the number of milliseconds associated with
+ * the provided duration. This duration of time is independent of any
+ * particular timezone or date.
+ */
ConvertedType::INTERVAL
};
const char* _kConvertedTypeNames[] = {
+ /**
+ * a BYTE_ARRAY actually contains UTF8 encoded chars
+ */
"UTF8",
+ /**
+ * a map is converted as an optional field containing a repeated key/value
pair
+ */
"MAP",
+ /**
+ * a key/value pair is converted into a group of two fields
+ */
"MAP_KEY_VALUE",
+ /**
+ * a list is converted into an optional field containing a repeated field
for its
+ * values
+ */
"LIST",
+ /**
+ * an enum is converted into a binary field
+ */
"ENUM",
+ /**
+ * A decimal value.
+ *
+ * This may be used to annotate binary or fixed primitive types. The
+ * underlying byte array stores the unscaled value encoded as two's
+ * complement using big-endian byte order (the most significant byte is the
+ * zeroth element). The value of the decimal is the value * 10^{-scale}.
+ *
+ * This must be accompanied by a (maximum) precision and a scale in the
+ * SchemaElement. The precision specifies the number of digits in the decimal
+ * and the scale stores the location of the decimal point. For example 1.23
+ * would have precision 3 (3 total digits) and scale 2 (the decimal point is
+ * 2 digits over).
+ */
"DECIMAL",
+ /**
+ * A Date
+ *
+ * Stored as days since Unix epoch, encoded as the INT32 physical type.
+ *
+ */
"DATE",
+ /**
+ * A time
+ *
+ * The total number of milliseconds since midnight. The value is stored
+ * as an INT32 physical type.
+ */
"TIME_MILLIS",
+ /**
+ * A time.
+ *
+ * The total number of microseconds since midnight. The value is stored as
+ * an INT64 physical type.
+ */
"TIME_MICROS",
+ /**
+ * A date/time combination
+ *
+ * Date and time recorded as milliseconds since the Unix epoch. Recorded as
+ * a physical type of INT64.
+ */
"TIMESTAMP_MILLIS",
+ /**
+ * A date/time combination
+ *
+ * Date and time recorded as microseconds since the Unix epoch. The value is
+ * stored as an INT64 physical type.
+ */
"TIMESTAMP_MICROS",
+ /**
+ * An unsigned integer value.
+ *
+ * The number describes the maximum number of meaningful data bits in
+ * the stored value. 8, 16 and 32 bit values are stored using the
+ * INT32 physical type. 64 bit values are stored using the INT64
+ * physical type.
+ *
+ */
"UINT_8",
"UINT_16",
"UINT_32",
"UINT_64",
+ /**
+ * A signed integer value.
+ *
+ * The number describes the maximum number of meaningful data bits in
+ * the stored value. 8, 16 and 32 bit values are stored using the
+ * INT32 physical type. 64 bit values are stored using the INT64
+ * physical type.
+ *
+ */
"INT_8",
"INT_16",
"INT_32",
"INT_64",
+ /**
+ * An embedded JSON document
+ *
+ * A JSON document embedded within a single UTF8 column.
+ */
"JSON",
+ /**
+ * An embedded BSON document
+ *
+ * A BSON document embedded within a single BINARY column.
+ */
"BSON",
+ /**
+ * An interval of time
+ *
+ * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12
+ * This data is composed of three separate little endian unsigned
+ * integers. Each stores a component of a duration of time. The first
+ * integer identifies the number of months associated with the duration,
+ * the second identifies the number of days associated with the duration
+ * and the third identifies the number of milliseconds associated with
+ * the provided duration. This duration of time is independent of any
+ * particular timezone or date.
+ */
"INTERVAL"
};
-const std::map<int, const char*>
_ConvertedType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(22,
_kConvertedTypeValues, _kConvertedTypeNames),
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_ConvertedType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(22,
_kConvertedTypeValues, _kConvertedTypeNames),
::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const ConvertedType::type& val) {
std::map<int, const char*>::const_iterator it =
_ConvertedType_VALUES_TO_NAMES.find(val);
@@ -124,16 +324,34 @@ std::string to_string(const ConvertedType::type& val) {
}
int _kFieldRepetitionTypeValues[] = {
+ /**
+ * This field is required (can not be null) and each record has exactly 1
value.
+ */
FieldRepetitionType::REQUIRED,
+ /**
+ * The field is optional (can be null) and each record has 0 or 1 values.
+ */
FieldRepetitionType::OPTIONAL,
+ /**
+ * The field is repeated and can contain 0 or more values
+ */
FieldRepetitionType::REPEATED
};
const char* _kFieldRepetitionTypeNames[] = {
+ /**
+ * This field is required (can not be null) and each record has exactly 1
value.
+ */
"REQUIRED",
+ /**
+ * The field is optional (can be null) and each record has 0 or 1 values.
+ */
"OPTIONAL",
+ /**
+ * The field is repeated and can contain 0 or more values
+ */
"REPEATED"
};
-const std::map<int, const char*>
_FieldRepetitionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3,
_kFieldRepetitionTypeValues, _kFieldRepetitionTypeNames),
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_FieldRepetitionType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3,
_kFieldRepetitionTypeValues, _kFieldRepetitionTypeNames),
::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const FieldRepetitionType::type&
val) {
std::map<int, const char*>::const_iterator it =
_FieldRepetitionType_VALUES_TO_NAMES.find(val);
@@ -155,28 +373,122 @@ std::string to_string(const FieldRepetitionType::type&
val) {
}
int _kEncodingValues[] = {
+ /**
+ * Default encoding.
+ * BOOLEAN - 1 bit per value. 0 is false; 1 is true.
+ * INT32 - 4 bytes per value. Stored as little-endian.
+ * INT64 - 8 bytes per value. Stored as little-endian.
+ * FLOAT - 4 bytes per value. IEEE. Stored as little-endian.
+ * DOUBLE - 8 bytes per value. IEEE. Stored as little-endian.
+ * BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
+ * FIXED_LEN_BYTE_ARRAY - Just the bytes.
+ */
Encoding::PLAIN,
+ /**
+ * Deprecated: Dictionary encoding. The values in the dictionary are encoded
in the
+ * plain type.
+ * in a data page use RLE_DICTIONARY instead.
+ * in a Dictionary page use PLAIN instead
+ */
Encoding::PLAIN_DICTIONARY,
+ /**
+ * Group packed run length encoding. Usable for definition/repetition levels
+ * encoding and Booleans (on one bit: 0 is false; 1 is true.)
+ */
Encoding::RLE,
+ /**
+ * Bit packed encoding. This can only be used if the data has a known max
+ * width. Usable for definition/repetition levels encoding.
+ */
Encoding::BIT_PACKED,
+ /**
+ * Delta encoding for integers. This can be used for int columns and works
best
+ * on sorted data
+ */
Encoding::DELTA_BINARY_PACKED,
+ /**
+ * Encoding for byte arrays to separate the length values and the data. The
lengths
+ * are encoded using DELTA_BINARY_PACKED
+ */
Encoding::DELTA_LENGTH_BYTE_ARRAY,
+ /**
+ * Incremental-encoded byte array. Prefix lengths are encoded using
DELTA_BINARY_PACKED.
+ * Suffixes are stored as delta length byte arrays.
+ */
Encoding::DELTA_BYTE_ARRAY,
+ /**
+ * Dictionary encoding: the ids are encoded using the RLE encoding
+ */
Encoding::RLE_DICTIONARY,
+ /**
+ * Encoding for floating-point data.
+ * K byte-streams are created where K is the size in bytes of the data type.
+ * The individual bytes of an FP value are scattered to the corresponding
stream and
+ * the streams are concatenated.
+ * This itself does not reduce the size of the data but can lead to better
compression
+ * afterwards.
+ */
Encoding::BYTE_STREAM_SPLIT
};
const char* _kEncodingNames[] = {
+ /**
+ * Default encoding.
+ * BOOLEAN - 1 bit per value. 0 is false; 1 is true.
+ * INT32 - 4 bytes per value. Stored as little-endian.
+ * INT64 - 8 bytes per value. Stored as little-endian.
+ * FLOAT - 4 bytes per value. IEEE. Stored as little-endian.
+ * DOUBLE - 8 bytes per value. IEEE. Stored as little-endian.
+ * BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
+ * FIXED_LEN_BYTE_ARRAY - Just the bytes.
+ */
"PLAIN",
+ /**
+ * Deprecated: Dictionary encoding. The values in the dictionary are encoded
in the
+ * plain type.
+ * in a data page use RLE_DICTIONARY instead.
+ * in a Dictionary page use PLAIN instead
+ */
"PLAIN_DICTIONARY",
+ /**
+ * Group packed run length encoding. Usable for definition/repetition levels
+ * encoding and Booleans (on one bit: 0 is false; 1 is true.)
+ */
"RLE",
+ /**
+ * Bit packed encoding. This can only be used if the data has a known max
+ * width. Usable for definition/repetition levels encoding.
+ */
"BIT_PACKED",
+ /**
+ * Delta encoding for integers. This can be used for int columns and works
best
+ * on sorted data
+ */
"DELTA_BINARY_PACKED",
+ /**
+ * Encoding for byte arrays to separate the length values and the data. The
lengths
+ * are encoded using DELTA_BINARY_PACKED
+ */
"DELTA_LENGTH_BYTE_ARRAY",
+ /**
+ * Incremental-encoded byte array. Prefix lengths are encoded using
DELTA_BINARY_PACKED.
+ * Suffixes are stored as delta length byte arrays.
+ */
"DELTA_BYTE_ARRAY",
+ /**
+ * Dictionary encoding: the ids are encoded using the RLE encoding
+ */
"RLE_DICTIONARY",
+ /**
+ * Encoding for floating-point data.
+ * K byte-streams are created where K is the size in bytes of the data type.
+ * The individual bytes of an FP value are scattered to the corresponding
stream and
+ * the streams are concatenated.
+ * This itself does not reduce the size of the data but can lead to better
compression
+ * afterwards.
+ */
"BYTE_STREAM_SPLIT"
};
-const std::map<int, const char*>
_Encoding_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(9, _kEncodingValues,
_kEncodingNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_Encoding_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(9, _kEncodingValues,
_kEncodingNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const Encoding::type& val) {
std::map<int, const char*>::const_iterator it =
_Encoding_VALUES_TO_NAMES.find(val);
@@ -217,7 +529,7 @@ const char* _kCompressionCodecNames[] = {
"ZSTD",
"LZ4_RAW"
};
-const std::map<int, const char*>
_CompressionCodec_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(8,
_kCompressionCodecValues, _kCompressionCodecNames),
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_CompressionCodec_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(8,
_kCompressionCodecValues, _kCompressionCodecNames),
::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const CompressionCodec::type& val)
{
std::map<int, const char*>::const_iterator it =
_CompressionCodec_VALUES_TO_NAMES.find(val);
@@ -250,7 +562,7 @@ const char* _kPageTypeNames[] = {
"DICTIONARY_PAGE",
"DATA_PAGE_V2"
};
-const std::map<int, const char*>
_PageType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kPageTypeValues,
_kPageTypeNames), ::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_PageType_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(4, _kPageTypeValues,
_kPageTypeNames), ::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const PageType::type& val) {
std::map<int, const char*>::const_iterator it =
_PageType_VALUES_TO_NAMES.find(val);
@@ -281,7 +593,7 @@ const char* _kBoundaryOrderNames[] = {
"ASCENDING",
"DESCENDING"
};
-const std::map<int, const char*>
_BoundaryOrder_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3,
_kBoundaryOrderValues, _kBoundaryOrderNames),
::apache::thrift::TEnumIterator(-1, NULL, NULL));
+const std::map<int, const char*>
_BoundaryOrder_VALUES_TO_NAMES(::apache::thrift::TEnumIterator(3,
_kBoundaryOrderValues, _kBoundaryOrderNames),
::apache::thrift::TEnumIterator(-1, nullptr, nullptr));
std::ostream& operator<<(std::ostream& out, const BoundaryOrder::type& val) {
std::map<int, const char*>::const_iterator it =
_BoundaryOrder_VALUES_TO_NAMES.find(val);
@@ -561,10 +873,10 @@ void swap(StringType &a, StringType &b) {
(void) b;
}
-StringType::StringType(const StringType& other2) {
+StringType::StringType(const StringType& other2) noexcept {
(void) other2;
}
-StringType& StringType::operator=(const StringType& other3) {
+StringType& StringType::operator=(const StringType& other3) noexcept {
(void) other3;
return *this;
}
@@ -629,10 +941,10 @@ void swap(UUIDType &a, UUIDType &b) {
(void) b;
}
-UUIDType::UUIDType(const UUIDType& other4) {
+UUIDType::UUIDType(const UUIDType& other4) noexcept {
(void) other4;
}
-UUIDType& UUIDType::operator=(const UUIDType& other5) {
+UUIDType& UUIDType::operator=(const UUIDType& other5) noexcept {
(void) other5;
return *this;
}
@@ -697,10 +1009,10 @@ void swap(MapType &a, MapType &b) {
(void) b;
}
-MapType::MapType(const MapType& other6) {
+MapType::MapType(const MapType& other6) noexcept {
(void) other6;
}
-MapType& MapType::operator=(const MapType& other7) {
+MapType& MapType::operator=(const MapType& other7) noexcept {
(void) other7;
return *this;
}
@@ -765,10 +1077,10 @@ void swap(ListType &a, ListType &b) {
(void) b;
}
-ListType::ListType(const ListType& other8) {
+ListType::ListType(const ListType& other8) noexcept {
(void) other8;
}
-ListType& ListType::operator=(const ListType& other9) {
+ListType& ListType::operator=(const ListType& other9) noexcept {
(void) other9;
return *this;
}
@@ -833,10 +1145,10 @@ void swap(EnumType &a, EnumType &b) {
(void) b;
}
-EnumType::EnumType(const EnumType& other10) {
+EnumType::EnumType(const EnumType& other10) noexcept {
(void) other10;
}
-EnumType& EnumType::operator=(const EnumType& other11) {
+EnumType& EnumType::operator=(const EnumType& other11) noexcept {
(void) other11;
return *this;
}
@@ -901,10 +1213,10 @@ void swap(DateType &a, DateType &b) {
(void) b;
}
-DateType::DateType(const DateType& other12) {
+DateType::DateType(const DateType& other12) noexcept {
(void) other12;
}
-DateType& DateType::operator=(const DateType& other13) {
+DateType& DateType::operator=(const DateType& other13) noexcept {
(void) other13;
return *this;
}
@@ -969,10 +1281,10 @@ void swap(NullType &a, NullType &b) {
(void) b;
}
-NullType::NullType(const NullType& other14) {
+NullType::NullType(const NullType& other14) noexcept {
(void) other14;
}
-NullType& NullType::operator=(const NullType& other15) {
+NullType& NullType::operator=(const NullType& other15) noexcept {
(void) other15;
return *this;
}
@@ -1080,11 +1392,11 @@ void swap(DecimalType &a, DecimalType &b) {
swap(a.precision, b.precision);
}
-DecimalType::DecimalType(const DecimalType& other16) {
+DecimalType::DecimalType(const DecimalType& other16) noexcept {
scale = other16.scale;
precision = other16.precision;
}
-DecimalType& DecimalType::operator=(const DecimalType& other17) {
+DecimalType& DecimalType::operator=(const DecimalType& other17) noexcept {
scale = other17.scale;
precision = other17.precision;
return *this;
@@ -1152,10 +1464,10 @@ void swap(MilliSeconds &a, MilliSeconds &b) {
(void) b;
}
-MilliSeconds::MilliSeconds(const MilliSeconds& other18) {
+MilliSeconds::MilliSeconds(const MilliSeconds& other18) noexcept {
(void) other18;
}
-MilliSeconds& MilliSeconds::operator=(const MilliSeconds& other19) {
+MilliSeconds& MilliSeconds::operator=(const MilliSeconds& other19) noexcept {
(void) other19;
return *this;
}
@@ -1220,10 +1532,10 @@ void swap(MicroSeconds &a, MicroSeconds &b) {
(void) b;
}
-MicroSeconds::MicroSeconds(const MicroSeconds& other20) {
+MicroSeconds::MicroSeconds(const MicroSeconds& other20) noexcept {
(void) other20;
}
-MicroSeconds& MicroSeconds::operator=(const MicroSeconds& other21) {
+MicroSeconds& MicroSeconds::operator=(const MicroSeconds& other21) noexcept {
(void) other21;
return *this;
}
@@ -1288,10 +1600,10 @@ void swap(NanoSeconds &a, NanoSeconds &b) {
(void) b;
}
-NanoSeconds::NanoSeconds(const NanoSeconds& other22) {
+NanoSeconds::NanoSeconds(const NanoSeconds& other22) noexcept {
(void) other22;
}
-NanoSeconds& NanoSeconds::operator=(const NanoSeconds& other23) {
+NanoSeconds& NanoSeconds::operator=(const NanoSeconds& other23) noexcept {
(void) other23;
return *this;
}
@@ -1417,13 +1729,13 @@ void swap(TimeUnit &a, TimeUnit &b) {
swap(a.__isset, b.__isset);
}
-TimeUnit::TimeUnit(const TimeUnit& other24) {
+TimeUnit::TimeUnit(const TimeUnit& other24) noexcept {
MILLIS = other24.MILLIS;
MICROS = other24.MICROS;
NANOS = other24.NANOS;
__isset = other24.__isset;
}
-TimeUnit& TimeUnit::operator=(const TimeUnit& other25) {
+TimeUnit& TimeUnit::operator=(const TimeUnit& other25) noexcept {
MILLIS = other25.MILLIS;
MICROS = other25.MICROS;
NANOS = other25.NANOS;
@@ -1537,11 +1849,11 @@ void swap(TimestampType &a, TimestampType &b) {
swap(a.unit, b.unit);
}
-TimestampType::TimestampType(const TimestampType& other26) {
+TimestampType::TimestampType(const TimestampType& other26) noexcept {
isAdjustedToUTC = other26.isAdjustedToUTC;
unit = other26.unit;
}
-TimestampType& TimestampType::operator=(const TimestampType& other27) {
+TimestampType& TimestampType::operator=(const TimestampType& other27) noexcept
{
isAdjustedToUTC = other27.isAdjustedToUTC;
unit = other27.unit;
return *this;
@@ -1652,11 +1964,11 @@ void swap(TimeType &a, TimeType &b) {
swap(a.unit, b.unit);
}
-TimeType::TimeType(const TimeType& other28) {
+TimeType::TimeType(const TimeType& other28) noexcept {
isAdjustedToUTC = other28.isAdjustedToUTC;
unit = other28.unit;
}
-TimeType& TimeType::operator=(const TimeType& other29) {
+TimeType& TimeType::operator=(const TimeType& other29) noexcept {
isAdjustedToUTC = other29.isAdjustedToUTC;
unit = other29.unit;
return *this;
@@ -1767,11 +2079,11 @@ void swap(IntType &a, IntType &b) {
swap(a.isSigned, b.isSigned);
}
-IntType::IntType(const IntType& other30) {
+IntType::IntType(const IntType& other30) noexcept {
bitWidth = other30.bitWidth;
isSigned = other30.isSigned;
}
-IntType& IntType::operator=(const IntType& other31) {
+IntType& IntType::operator=(const IntType& other31) noexcept {
bitWidth = other31.bitWidth;
isSigned = other31.isSigned;
return *this;
@@ -1839,10 +2151,10 @@ void swap(JsonType &a, JsonType &b) {
(void) b;
}
-JsonType::JsonType(const JsonType& other32) {
+JsonType::JsonType(const JsonType& other32) noexcept {
(void) other32;
}
-JsonType& JsonType::operator=(const JsonType& other33) {
+JsonType& JsonType::operator=(const JsonType& other33) noexcept {
(void) other33;
return *this;
}
@@ -1907,10 +2219,10 @@ void swap(BsonType &a, BsonType &b) {
(void) b;
}
-BsonType::BsonType(const BsonType& other34) {
+BsonType::BsonType(const BsonType& other34) noexcept {
(void) other34;
}
-BsonType& BsonType::operator=(const BsonType& other35) {
+BsonType& BsonType::operator=(const BsonType& other35) noexcept {
(void) other35;
return *this;
}
@@ -2226,7 +2538,7 @@ void swap(LogicalType &a, LogicalType &b) {
swap(a.__isset, b.__isset);
}
-LogicalType::LogicalType(const LogicalType& other36) {
+LogicalType::LogicalType(const LogicalType& other36) noexcept {
STRING = other36.STRING;
MAP = other36.MAP;
LIST = other36.LIST;
@@ -2242,7 +2554,7 @@ LogicalType::LogicalType(const LogicalType& other36) {
UUID = other36.UUID;
__isset = other36.__isset;
}
-LogicalType& LogicalType::operator=(const LogicalType& other37) {
+LogicalType& LogicalType::operator=(const LogicalType& other37) noexcept {
STRING = other37.STRING;
MAP = other37.MAP;
LIST = other37.LIST;
@@ -2364,7 +2676,7 @@ uint32_t
SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast38;
xfer += iprot->readI32(ecast38);
- this->type = (Type::type)ecast38;
+ this->type = static_cast<Type::type>(ecast38);
this->__isset.type = true;
} else {
xfer += iprot->skip(ftype);
@@ -2382,7 +2694,7 @@ uint32_t
SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast39;
xfer += iprot->readI32(ecast39);
- this->repetition_type = (FieldRepetitionType::type)ecast39;
+ this->repetition_type =
static_cast<FieldRepetitionType::type>(ecast39);
this->__isset.repetition_type = true;
} else {
xfer += iprot->skip(ftype);
@@ -2408,7 +2720,7 @@ uint32_t
SchemaElement::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast40;
xfer += iprot->readI32(ecast40);
- this->converted_type = (ConvertedType::type)ecast40;
+ this->converted_type = static_cast<ConvertedType::type>(ecast40);
this->__isset.converted_type = true;
} else {
xfer += iprot->skip(ftype);
@@ -2467,7 +2779,7 @@ uint32_t
SchemaElement::write(::apache::thrift::protocol::TProtocol* oprot) cons
if (this->__isset.type) {
xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_I32,
1);
- xfer += oprot->writeI32((int32_t)this->type);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->type));
xfer += oprot->writeFieldEnd();
}
if (this->__isset.type_length) {
@@ -2477,7 +2789,7 @@ uint32_t
SchemaElement::write(::apache::thrift::protocol::TProtocol* oprot) cons
}
if (this->__isset.repetition_type) {
xfer += oprot->writeFieldBegin("repetition_type",
::apache::thrift::protocol::T_I32, 3);
- xfer += oprot->writeI32((int32_t)this->repetition_type);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->repetition_type));
xfer += oprot->writeFieldEnd();
}
xfer += oprot->writeFieldBegin("name", ::apache::thrift::protocol::T_STRING,
4);
@@ -2491,7 +2803,7 @@ uint32_t
SchemaElement::write(::apache::thrift::protocol::TProtocol* oprot) cons
}
if (this->__isset.converted_type) {
xfer += oprot->writeFieldBegin("converted_type",
::apache::thrift::protocol::T_I32, 6);
- xfer += oprot->writeI32((int32_t)this->converted_type);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->converted_type));
xfer += oprot->writeFieldEnd();
}
if (this->__isset.scale) {
@@ -2646,7 +2958,7 @@ uint32_t
DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast43;
xfer += iprot->readI32(ecast43);
- this->encoding = (Encoding::type)ecast43;
+ this->encoding = static_cast<Encoding::type>(ecast43);
isset_encoding = true;
} else {
xfer += iprot->skip(ftype);
@@ -2656,7 +2968,7 @@ uint32_t
DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast44;
xfer += iprot->readI32(ecast44);
- this->definition_level_encoding = (Encoding::type)ecast44;
+ this->definition_level_encoding =
static_cast<Encoding::type>(ecast44);
isset_definition_level_encoding = true;
} else {
xfer += iprot->skip(ftype);
@@ -2666,7 +2978,7 @@ uint32_t
DataPageHeader::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast45;
xfer += iprot->readI32(ecast45);
- this->repetition_level_encoding = (Encoding::type)ecast45;
+ this->repetition_level_encoding =
static_cast<Encoding::type>(ecast45);
isset_repetition_level_encoding = true;
} else {
xfer += iprot->skip(ftype);
@@ -2710,15 +3022,15 @@ uint32_t
DataPageHeader::write(::apache::thrift::protocol::TProtocol* oprot) con
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("encoding",
::apache::thrift::protocol::T_I32, 2);
- xfer += oprot->writeI32((int32_t)this->encoding);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->encoding));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("definition_level_encoding",
::apache::thrift::protocol::T_I32, 3);
- xfer += oprot->writeI32((int32_t)this->definition_level_encoding);
+ xfer +=
oprot->writeI32(static_cast<int32_t>(this->definition_level_encoding));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("repetition_level_encoding",
::apache::thrift::protocol::T_I32, 4);
- xfer += oprot->writeI32((int32_t)this->repetition_level_encoding);
+ xfer +=
oprot->writeI32(static_cast<int32_t>(this->repetition_level_encoding));
xfer += oprot->writeFieldEnd();
if (this->__isset.statistics) {
@@ -2824,10 +3136,10 @@ void swap(IndexPageHeader &a, IndexPageHeader &b) {
(void) b;
}
-IndexPageHeader::IndexPageHeader(const IndexPageHeader& other48) {
+IndexPageHeader::IndexPageHeader(const IndexPageHeader& other48) noexcept {
(void) other48;
}
-IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other49) {
+IndexPageHeader& IndexPageHeader::operator=(const IndexPageHeader& other49)
noexcept {
(void) other49;
return *this;
}
@@ -2896,7 +3208,7 @@ uint32_t
DictionaryPageHeader::read(::apache::thrift::protocol::TProtocol* iprot
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast50;
xfer += iprot->readI32(ecast50);
- this->encoding = (Encoding::type)ecast50;
+ this->encoding = static_cast<Encoding::type>(ecast50);
isset_encoding = true;
} else {
xfer += iprot->skip(ftype);
@@ -2936,7 +3248,7 @@ uint32_t
DictionaryPageHeader::write(::apache::thrift::protocol::TProtocol* opro
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("encoding",
::apache::thrift::protocol::T_I32, 2);
- xfer += oprot->writeI32((int32_t)this->encoding);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->encoding));
xfer += oprot->writeFieldEnd();
if (this->__isset.is_sorted) {
@@ -2957,13 +3269,13 @@ void swap(DictionaryPageHeader &a, DictionaryPageHeader
&b) {
swap(a.__isset, b.__isset);
}
-DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader&
other51) {
+DictionaryPageHeader::DictionaryPageHeader(const DictionaryPageHeader&
other51) noexcept {
num_values = other51.num_values;
encoding = other51.encoding;
is_sorted = other51.is_sorted;
__isset = other51.__isset;
}
-DictionaryPageHeader& DictionaryPageHeader::operator=(const
DictionaryPageHeader& other52) {
+DictionaryPageHeader& DictionaryPageHeader::operator=(const
DictionaryPageHeader& other52) noexcept {
num_values = other52.num_values;
encoding = other52.encoding;
is_sorted = other52.is_sorted;
@@ -3079,7 +3391,7 @@ uint32_t
DataPageHeaderV2::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast53;
xfer += iprot->readI32(ecast53);
- this->encoding = (Encoding::type)ecast53;
+ this->encoding = static_cast<Encoding::type>(ecast53);
isset_encoding = true;
} else {
xfer += iprot->skip(ftype);
@@ -3159,7 +3471,7 @@ uint32_t
DataPageHeaderV2::write(::apache::thrift::protocol::TProtocol* oprot) c
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("encoding",
::apache::thrift::protocol::T_I32, 4);
- xfer += oprot->writeI32((int32_t)this->encoding);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->encoding));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("definition_levels_byte_length",
::apache::thrift::protocol::T_I32, 5);
@@ -3290,10 +3602,10 @@ void swap(SplitBlockAlgorithm &a, SplitBlockAlgorithm
&b) {
(void) b;
}
-SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other56) {
+SplitBlockAlgorithm::SplitBlockAlgorithm(const SplitBlockAlgorithm& other56)
noexcept {
(void) other56;
}
-SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm&
other57) {
+SplitBlockAlgorithm& SplitBlockAlgorithm::operator=(const SplitBlockAlgorithm&
other57) noexcept {
(void) other57;
return *this;
}
@@ -3381,11 +3693,11 @@ void swap(BloomFilterAlgorithm &a, BloomFilterAlgorithm
&b) {
swap(a.__isset, b.__isset);
}
-BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm&
other58) {
+BloomFilterAlgorithm::BloomFilterAlgorithm(const BloomFilterAlgorithm&
other58) noexcept {
BLOCK = other58.BLOCK;
__isset = other58.__isset;
}
-BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const
BloomFilterAlgorithm& other59) {
+BloomFilterAlgorithm& BloomFilterAlgorithm::operator=(const
BloomFilterAlgorithm& other59) noexcept {
BLOCK = other59.BLOCK;
__isset = other59.__isset;
return *this;
@@ -3452,10 +3764,10 @@ void swap(XxHash &a, XxHash &b) {
(void) b;
}
-XxHash::XxHash(const XxHash& other60) {
+XxHash::XxHash(const XxHash& other60) noexcept {
(void) other60;
}
-XxHash& XxHash::operator=(const XxHash& other61) {
+XxHash& XxHash::operator=(const XxHash& other61) noexcept {
(void) other61;
return *this;
}
@@ -3543,11 +3855,11 @@ void swap(BloomFilterHash &a, BloomFilterHash &b) {
swap(a.__isset, b.__isset);
}
-BloomFilterHash::BloomFilterHash(const BloomFilterHash& other62) {
+BloomFilterHash::BloomFilterHash(const BloomFilterHash& other62) noexcept {
XXHASH = other62.XXHASH;
__isset = other62.__isset;
}
-BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other63) {
+BloomFilterHash& BloomFilterHash::operator=(const BloomFilterHash& other63)
noexcept {
XXHASH = other63.XXHASH;
__isset = other63.__isset;
return *this;
@@ -3614,10 +3926,10 @@ void swap(Uncompressed &a, Uncompressed &b) {
(void) b;
}
-Uncompressed::Uncompressed(const Uncompressed& other64) {
+Uncompressed::Uncompressed(const Uncompressed& other64) noexcept {
(void) other64;
}
-Uncompressed& Uncompressed::operator=(const Uncompressed& other65) {
+Uncompressed& Uncompressed::operator=(const Uncompressed& other65) noexcept {
(void) other65;
return *this;
}
@@ -3705,11 +4017,11 @@ void swap(BloomFilterCompression &a,
BloomFilterCompression &b) {
swap(a.__isset, b.__isset);
}
-BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression&
other66) {
+BloomFilterCompression::BloomFilterCompression(const BloomFilterCompression&
other66) noexcept {
UNCOMPRESSED = other66.UNCOMPRESSED;
__isset = other66.__isset;
}
-BloomFilterCompression& BloomFilterCompression::operator=(const
BloomFilterCompression& other67) {
+BloomFilterCompression& BloomFilterCompression::operator=(const
BloomFilterCompression& other67) noexcept {
UNCOMPRESSED = other67.UNCOMPRESSED;
__isset = other67.__isset;
return *this;
@@ -3859,13 +4171,13 @@ void swap(BloomFilterHeader &a, BloomFilterHeader &b) {
swap(a.compression, b.compression);
}
-BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other68) {
+BloomFilterHeader::BloomFilterHeader(const BloomFilterHeader& other68)
noexcept {
numBytes = other68.numBytes;
algorithm = other68.algorithm;
hash = other68.hash;
compression = other68.compression;
}
-BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader&
other69) {
+BloomFilterHeader& BloomFilterHeader::operator=(const BloomFilterHeader&
other69) noexcept {
numBytes = other69.numBytes;
algorithm = other69.algorithm;
hash = other69.hash;
@@ -3958,7 +4270,7 @@ uint32_t
PageHeader::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast70;
xfer += iprot->readI32(ecast70);
- this->type = (PageType::type)ecast70;
+ this->type = static_cast<PageType::type>(ecast70);
isset_type = true;
} else {
xfer += iprot->skip(ftype);
@@ -4044,7 +4356,7 @@ uint32_t
PageHeader::write(::apache::thrift::protocol::TProtocol* oprot) const {
xfer += oprot->writeStructBegin("PageHeader");
xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_I32, 1);
- xfer += oprot->writeI32((int32_t)this->type);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->type));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("uncompressed_page_size",
::apache::thrift::protocol::T_I32, 2);
@@ -4370,12 +4682,12 @@ void swap(SortingColumn &a, SortingColumn &b) {
swap(a.nulls_first, b.nulls_first);
}
-SortingColumn::SortingColumn(const SortingColumn& other75) {
+SortingColumn::SortingColumn(const SortingColumn& other75) noexcept {
column_idx = other75.column_idx;
descending = other75.descending;
nulls_first = other75.nulls_first;
}
-SortingColumn& SortingColumn::operator=(const SortingColumn& other76) {
+SortingColumn& SortingColumn::operator=(const SortingColumn& other76) noexcept
{
column_idx = other76.column_idx;
descending = other76.descending;
nulls_first = other76.nulls_first;
@@ -4441,7 +4753,7 @@ uint32_t
PageEncodingStats::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast77;
xfer += iprot->readI32(ecast77);
- this->page_type = (PageType::type)ecast77;
+ this->page_type = static_cast<PageType::type>(ecast77);
isset_page_type = true;
} else {
xfer += iprot->skip(ftype);
@@ -4451,7 +4763,7 @@ uint32_t
PageEncodingStats::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast78;
xfer += iprot->readI32(ecast78);
- this->encoding = (Encoding::type)ecast78;
+ this->encoding = static_cast<Encoding::type>(ecast78);
isset_encoding = true;
} else {
xfer += iprot->skip(ftype);
@@ -4489,11 +4801,11 @@ uint32_t
PageEncodingStats::write(::apache::thrift::protocol::TProtocol* oprot)
xfer += oprot->writeStructBegin("PageEncodingStats");
xfer += oprot->writeFieldBegin("page_type",
::apache::thrift::protocol::T_I32, 1);
- xfer += oprot->writeI32((int32_t)this->page_type);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->page_type));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("encoding",
::apache::thrift::protocol::T_I32, 2);
- xfer += oprot->writeI32((int32_t)this->encoding);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->encoding));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("count", ::apache::thrift::protocol::T_I32,
3);
@@ -4512,12 +4824,12 @@ void swap(PageEncodingStats &a, PageEncodingStats &b) {
swap(a.count, b.count);
}
-PageEncodingStats::PageEncodingStats(const PageEncodingStats& other79) {
+PageEncodingStats::PageEncodingStats(const PageEncodingStats& other79)
noexcept {
page_type = other79.page_type;
encoding = other79.encoding;
count = other79.count;
}
-PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats&
other80) {
+PageEncodingStats& PageEncodingStats::operator=(const PageEncodingStats&
other80) noexcept {
page_type = other80.page_type;
encoding = other80.encoding;
count = other80.count;
@@ -4638,7 +4950,7 @@ uint32_t
ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast81;
xfer += iprot->readI32(ecast81);
- this->type = (Type::type)ecast81;
+ this->type = static_cast<Type::type>(ecast81);
isset_type = true;
} else {
xfer += iprot->skip(ftype);
@@ -4657,7 +4969,7 @@ uint32_t
ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) {
{
int32_t ecast87;
xfer += iprot->readI32(ecast87);
- this->encodings[_i86] = (Encoding::type)ecast87;
+ this->encodings[_i86] = static_cast<Encoding::type>(ecast87);
}
xfer += iprot->readListEnd();
}
@@ -4690,7 +5002,7 @@ uint32_t
ColumnMetaData::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast93;
xfer += iprot->readI32(ecast93);
- this->codec = (CompressionCodec::type)ecast93;
+ this->codec = static_cast<CompressionCodec::type>(ecast93);
isset_codec = true;
} else {
xfer += iprot->skip(ftype);
@@ -4834,7 +5146,7 @@ uint32_t
ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con
xfer += oprot->writeStructBegin("ColumnMetaData");
xfer += oprot->writeFieldBegin("type", ::apache::thrift::protocol::T_I32, 1);
- xfer += oprot->writeI32((int32_t)this->type);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->type));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("encodings",
::apache::thrift::protocol::T_LIST, 2);
@@ -4843,7 +5155,7 @@ uint32_t
ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con
std::vector<Encoding::type> ::const_iterator _iter104;
for (_iter104 = this->encodings.begin(); _iter104 !=
this->encodings.end(); ++_iter104)
{
- xfer += oprot->writeI32((int32_t)(*_iter104));
+ xfer += oprot->writeI32(static_cast<int32_t>((*_iter104)));
}
xfer += oprot->writeListEnd();
}
@@ -4862,7 +5174,7 @@ uint32_t
ColumnMetaData::write(::apache::thrift::protocol::TProtocol* oprot) con
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("codec", ::apache::thrift::protocol::T_I32,
4);
- xfer += oprot->writeI32((int32_t)this->codec);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->codec));
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("num_values",
::apache::thrift::protocol::T_I64, 5);
@@ -5061,10 +5373,10 @@ void swap(EncryptionWithFooterKey &a,
EncryptionWithFooterKey &b) {
(void) b;
}
-EncryptionWithFooterKey::EncryptionWithFooterKey(const
EncryptionWithFooterKey& other110) {
+EncryptionWithFooterKey::EncryptionWithFooterKey(const
EncryptionWithFooterKey& other110) noexcept {
(void) other110;
}
-EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const
EncryptionWithFooterKey& other111) {
+EncryptionWithFooterKey& EncryptionWithFooterKey::operator=(const
EncryptionWithFooterKey& other111) noexcept {
(void) other111;
return *this;
}
@@ -5922,10 +6234,10 @@ void swap(TypeDefinedOrder &a, TypeDefinedOrder &b) {
(void) b;
}
-TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other138) {
+TypeDefinedOrder::TypeDefinedOrder(const TypeDefinedOrder& other138) noexcept {
(void) other138;
}
-TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder&
other139) {
+TypeDefinedOrder& TypeDefinedOrder::operator=(const TypeDefinedOrder&
other139) noexcept {
(void) other139;
return *this;
}
@@ -6013,11 +6325,11 @@ void swap(ColumnOrder &a, ColumnOrder &b) {
swap(a.__isset, b.__isset);
}
-ColumnOrder::ColumnOrder(const ColumnOrder& other140) {
+ColumnOrder::ColumnOrder(const ColumnOrder& other140) noexcept {
TYPE_ORDER = other140.TYPE_ORDER;
__isset = other140.__isset;
}
-ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other141) {
+ColumnOrder& ColumnOrder::operator=(const ColumnOrder& other141) noexcept {
TYPE_ORDER = other141.TYPE_ORDER;
__isset = other141.__isset;
return *this;
@@ -6147,12 +6459,12 @@ void swap(PageLocation &a, PageLocation &b) {
swap(a.first_row_index, b.first_row_index);
}
-PageLocation::PageLocation(const PageLocation& other142) {
+PageLocation::PageLocation(const PageLocation& other142) noexcept {
offset = other142.offset;
compressed_page_size = other142.compressed_page_size;
first_row_index = other142.first_row_index;
}
-PageLocation& PageLocation::operator=(const PageLocation& other143) {
+PageLocation& PageLocation::operator=(const PageLocation& other143) noexcept {
offset = other143.offset;
compressed_page_size = other143.compressed_page_size;
first_row_index = other143.first_row_index;
@@ -6400,7 +6712,7 @@ uint32_t
ColumnIndex::read(::apache::thrift::protocol::TProtocol* iprot) {
if (ftype == ::apache::thrift::protocol::T_I32) {
int32_t ecast167;
xfer += iprot->readI32(ecast167);
- this->boundary_order = (BoundaryOrder::type)ecast167;
+ this->boundary_order = static_cast<BoundaryOrder::type>(ecast167);
isset_boundary_order = true;
} else {
xfer += iprot->skip(ftype);
@@ -6488,7 +6800,7 @@ uint32_t
ColumnIndex::write(::apache::thrift::protocol::TProtocol* oprot) const
xfer += oprot->writeFieldEnd();
xfer += oprot->writeFieldBegin("boundary_order",
::apache::thrift::protocol::T_I32, 4);
- xfer += oprot->writeI32((int32_t)this->boundary_order);
+ xfer += oprot->writeI32(static_cast<int32_t>(this->boundary_order));
xfer += oprot->writeFieldEnd();
if (this->__isset.null_counts) {
diff --git a/cpp/src/generated/parquet_types.h
b/cpp/src/generated/parquet_types.h
index 3d7edd4098..826e589304 100644
--- a/cpp/src/generated/parquet_types.h
+++ b/cpp/src/generated/parquet_types.h
@@ -1,5 +1,5 @@
/**
- * Autogenerated by Thrift Compiler (0.13.0)
+ * Autogenerated by Thrift Compiler (0.17.0)
*
* DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING
* @generated
@@ -22,6 +22,12 @@
namespace parquet { namespace format {
+/**
+ * Types supported by Parquet. These types are intended to be used in
combination
+ * with the encodings to control the on disk storage format.
+ * For example INT16 is not included as a type since a good encoding of INT32
+ * would handle this.
+ */
struct Type {
enum type {
BOOLEAN = 0,
@@ -41,29 +47,135 @@ std::ostream& operator<<(std::ostream& out, const
Type::type& val);
std::string to_string(const Type::type& val);
+/**
+ * DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet.
+ * ConvertedType is superseded by LogicalType. This enum should not be
extended.
+ *
+ * See LogicalTypes.md for conversion between ConvertedType and LogicalType.
+ */
struct ConvertedType {
enum type {
+ /**
+ * a BYTE_ARRAY actually contains UTF8 encoded chars
+ */
UTF8 = 0,
+ /**
+ * a map is converted as an optional field containing a repeated key/value
pair
+ */
MAP = 1,
+ /**
+ * a key/value pair is converted into a group of two fields
+ */
MAP_KEY_VALUE = 2,
+ /**
+ * a list is converted into an optional field containing a repeated field
for its
+ * values
+ */
LIST = 3,
+ /**
+ * an enum is converted into a binary field
+ */
ENUM = 4,
+ /**
+ * A decimal value.
+ *
+ * This may be used to annotate binary or fixed primitive types. The
+ * underlying byte array stores the unscaled value encoded as two's
+ * complement using big-endian byte order (the most significant byte is the
+ * zeroth element). The value of the decimal is the value * 10^{-scale}.
+ *
+ * This must be accompanied by a (maximum) precision and a scale in the
+ * SchemaElement. The precision specifies the number of digits in the
decimal
+ * and the scale stores the location of the decimal point. For example 1.23
+ * would have precision 3 (3 total digits) and scale 2 (the decimal point
is
+ * 2 digits over).
+ */
DECIMAL = 5,
+ /**
+ * A Date
+ *
+ * Stored as days since Unix epoch, encoded as the INT32 physical type.
+ *
+ */
DATE = 6,
+ /**
+ * A time
+ *
+ * The total number of milliseconds since midnight. The value is stored
+ * as an INT32 physical type.
+ */
TIME_MILLIS = 7,
+ /**
+ * A time.
+ *
+ * The total number of microseconds since midnight. The value is stored as
+ * an INT64 physical type.
+ */
TIME_MICROS = 8,
+ /**
+ * A date/time combination
+ *
+ * Date and time recorded as milliseconds since the Unix epoch. Recorded
as
+ * a physical type of INT64.
+ */
TIMESTAMP_MILLIS = 9,
+ /**
+ * A date/time combination
+ *
+ * Date and time recorded as microseconds since the Unix epoch. The value
is
+ * stored as an INT64 physical type.
+ */
TIMESTAMP_MICROS = 10,
+ /**
+ * An unsigned integer value.
+ *
+ * The number describes the maximum number of meaningful data bits in
+ * the stored value. 8, 16 and 32 bit values are stored using the
+ * INT32 physical type. 64 bit values are stored using the INT64
+ * physical type.
+ *
+ */
UINT_8 = 11,
UINT_16 = 12,
UINT_32 = 13,
UINT_64 = 14,
+ /**
+ * A signed integer value.
+ *
+ * The number describes the maximum number of meaningful data bits in
+ * the stored value. 8, 16 and 32 bit values are stored using the
+ * INT32 physical type. 64 bit values are stored using the INT64
+ * physical type.
+ *
+ */
INT_8 = 15,
INT_16 = 16,
INT_32 = 17,
INT_64 = 18,
+ /**
+ * An embedded JSON document
+ *
+ * A JSON document embedded within a single UTF8 column.
+ */
JSON = 19,
+ /**
+ * An embedded BSON document
+ *
+ * A BSON document embedded within a single BINARY column.
+ */
BSON = 20,
+ /**
+ * An interval of time
+ *
+ * This type annotates data stored as a FIXED_LEN_BYTE_ARRAY of length 12
+ * This data is composed of three separate little endian unsigned
+ * integers. Each stores a component of a duration of time. The first
+ * integer identifies the number of months associated with the duration,
+ * the second identifies the number of days associated with the duration
+ * and the third identifies the number of milliseconds associated with
+ * the provided duration. This duration of time is independent of any
+ * particular timezone or date.
+ */
INTERVAL = 21
};
};
@@ -74,10 +186,22 @@ std::ostream& operator<<(std::ostream& out, const
ConvertedType::type& val);
std::string to_string(const ConvertedType::type& val);
+/**
+ * Representation of Schemas
+ */
struct FieldRepetitionType {
enum type {
+ /**
+ * This field is required (can not be null) and each record has exactly 1
value.
+ */
REQUIRED = 0,
+ /**
+ * The field is optional (can be null) and each record has 0 or 1 values.
+ */
OPTIONAL = 1,
+ /**
+ * The field is repeated and can contain 0 or more values
+ */
REPEATED = 2
};
};
@@ -88,16 +212,68 @@ std::ostream& operator<<(std::ostream& out, const
FieldRepetitionType::type& val
std::string to_string(const FieldRepetitionType::type& val);
+/**
+ * Encodings supported by Parquet. Not all encodings are valid for all types.
These
+ * enums are also used to specify the encoding of definition and repetition
levels.
+ * See the accompanying doc for the details of the more complicated encodings.
+ */
struct Encoding {
enum type {
+ /**
+ * Default encoding.
+ * BOOLEAN - 1 bit per value. 0 is false; 1 is true.
+ * INT32 - 4 bytes per value. Stored as little-endian.
+ * INT64 - 8 bytes per value. Stored as little-endian.
+ * FLOAT - 4 bytes per value. IEEE. Stored as little-endian.
+ * DOUBLE - 8 bytes per value. IEEE. Stored as little-endian.
+ * BYTE_ARRAY - 4 byte length stored as little endian, followed by bytes.
+ * FIXED_LEN_BYTE_ARRAY - Just the bytes.
+ */
PLAIN = 0,
+ /**
+ * Deprecated: Dictionary encoding. The values in the dictionary are
encoded in the
+ * plain type.
+ * in a data page use RLE_DICTIONARY instead.
+ * in a Dictionary page use PLAIN instead
+ */
PLAIN_DICTIONARY = 2,
+ /**
+ * Group packed run length encoding. Usable for definition/repetition
levels
+ * encoding and Booleans (on one bit: 0 is false; 1 is true.)
+ */
RLE = 3,
+ /**
+ * Bit packed encoding. This can only be used if the data has a known max
+ * width. Usable for definition/repetition levels encoding.
+ */
BIT_PACKED = 4,
+ /**
+ * Delta encoding for integers. This can be used for int columns and works
best
+ * on sorted data
+ */
DELTA_BINARY_PACKED = 5,
+ /**
+ * Encoding for byte arrays to separate the length values and the data.
The lengths
+ * are encoded using DELTA_BINARY_PACKED
+ */
DELTA_LENGTH_BYTE_ARRAY = 6,
+ /**
+ * Incremental-encoded byte array. Prefix lengths are encoded using
DELTA_BINARY_PACKED.
+ * Suffixes are stored as delta length byte arrays.
+ */
DELTA_BYTE_ARRAY = 7,
+ /**
+ * Dictionary encoding: the ids are encoded using the RLE encoding
+ */
RLE_DICTIONARY = 8,
+ /**
+ * Encoding for floating-point data.
+ * K byte-streams are created where K is the size in bytes of the data
type.
+ * The individual bytes of an FP value are scattered to the corresponding
stream and
+ * the streams are concatenated.
+ * This itself does not reduce the size of the data but can lead to better
compression
+ * afterwards.
+ */
BYTE_STREAM_SPLIT = 9
};
};
@@ -108,6 +284,15 @@ std::ostream& operator<<(std::ostream& out, const
Encoding::type& val);
std::string to_string(const Encoding::type& val);
+/**
+ * Supported compression algorithms.
+ *
+ * Codecs added in format version X.Y can be read by readers based on X.Y and
later.
+ * Codec support may vary between readers based on the format version and
+ * libraries available at runtime.
+ *
+ * See Compression.md for a detailed specification of these algorithms.
+ */
struct CompressionCodec {
enum type {
UNCOMPRESSED = 0,
@@ -142,6 +327,10 @@ std::ostream& operator<<(std::ostream& out, const
PageType::type& val);
std::string to_string(const PageType::type& val);
+/**
+ * Enum to annotate whether lists of min/max elements inside ColumnIndex
+ * are ordered and if so, in which direction.
+ */
struct BoundaryOrder {
enum type {
UNORDERED = 0,
@@ -268,19 +457,54 @@ typedef struct _Statistics__isset {
bool min_value :1;
} _Statistics__isset;
+/**
+ * Statistics per row group and per page
+ * All fields are optional.
+ */
class Statistics : public virtual ::apache::thrift::TBase {
public:
Statistics(const Statistics&);
Statistics& operator=(const Statistics&);
- Statistics() : max(), min(), null_count(0), distinct_count(0), max_value(),
min_value() {
+ Statistics() noexcept
+ : max(),
+ min(),
+ null_count(0),
+ distinct_count(0),
+ max_value(),
+ min_value() {
}
virtual ~Statistics() noexcept;
+ /**
+ * DEPRECATED: min and max value of the column. Use min_value and max_value.
+ *
+ * Values are encoded using PLAIN encoding, except that variable-length byte
+ * arrays do not include a length prefix.
+ *
+ * These fields encode min and max values determined by signed comparison
+ * only. New files should use the correct order for a column's logical type
+ * and store the values in the min_value and max_value fields.
+ *
+ * To support older readers, these may be set when the column order is
+ * signed.
+ */
std::string max;
std::string min;
+ /**
+ * count of null value in the column
+ */
int64_t null_count;
+ /**
+ * count of distinct values occurring
+ */
int64_t distinct_count;
+ /**
+ * Min and max values for the column, determined by its ColumnOrder.
+ *
+ * Values are encoded using PLAIN encoding, except that variable-length byte
+ * arrays do not include a length prefix.
+ */
std::string max_value;
std::string min_value;
@@ -332,8 +556,8 @@ class Statistics : public virtual ::apache::thrift::TBase {
bool operator < (const Statistics & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -343,12 +567,15 @@ void swap(Statistics &a, Statistics &b);
std::ostream& operator<<(std::ostream& out, const Statistics& obj);
+/**
+ * Empty structs to use as logical type annotations
+ */
class StringType : public virtual ::apache::thrift::TBase {
public:
- StringType(const StringType&);
- StringType& operator=(const StringType&);
- StringType() {
+ StringType(const StringType&) noexcept;
+ StringType& operator=(const StringType&) noexcept;
+ StringType() noexcept {
}
virtual ~StringType() noexcept;
@@ -363,8 +590,8 @@ class StringType : public virtual ::apache::thrift::TBase {
bool operator < (const StringType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -377,9 +604,9 @@ std::ostream& operator<<(std::ostream& out, const
StringType& obj);
class UUIDType : public virtual ::apache::thrift::TBase {
public:
- UUIDType(const UUIDType&);
- UUIDType& operator=(const UUIDType&);
- UUIDType() {
+ UUIDType(const UUIDType&) noexcept;
+ UUIDType& operator=(const UUIDType&) noexcept;
+ UUIDType() noexcept {
}
virtual ~UUIDType() noexcept;
@@ -394,8 +621,8 @@ class UUIDType : public virtual ::apache::thrift::TBase {
bool operator < (const UUIDType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -408,9 +635,9 @@ std::ostream& operator<<(std::ostream& out, const UUIDType&
obj);
class MapType : public virtual ::apache::thrift::TBase {
public:
- MapType(const MapType&);
- MapType& operator=(const MapType&);
- MapType() {
+ MapType(const MapType&) noexcept;
+ MapType& operator=(const MapType&) noexcept;
+ MapType() noexcept {
}
virtual ~MapType() noexcept;
@@ -425,8 +652,8 @@ class MapType : public virtual ::apache::thrift::TBase {
bool operator < (const MapType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -439,9 +666,9 @@ std::ostream& operator<<(std::ostream& out, const MapType&
obj);
class ListType : public virtual ::apache::thrift::TBase {
public:
- ListType(const ListType&);
- ListType& operator=(const ListType&);
- ListType() {
+ ListType(const ListType&) noexcept;
+ ListType& operator=(const ListType&) noexcept;
+ ListType() noexcept {
}
virtual ~ListType() noexcept;
@@ -456,8 +683,8 @@ class ListType : public virtual ::apache::thrift::TBase {
bool operator < (const ListType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -470,9 +697,9 @@ std::ostream& operator<<(std::ostream& out, const ListType&
obj);
class EnumType : public virtual ::apache::thrift::TBase {
public:
- EnumType(const EnumType&);
- EnumType& operator=(const EnumType&);
- EnumType() {
+ EnumType(const EnumType&) noexcept;
+ EnumType& operator=(const EnumType&) noexcept;
+ EnumType() noexcept {
}
virtual ~EnumType() noexcept;
@@ -487,8 +714,8 @@ class EnumType : public virtual ::apache::thrift::TBase {
bool operator < (const EnumType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -501,9 +728,9 @@ std::ostream& operator<<(std::ostream& out, const EnumType&
obj);
class DateType : public virtual ::apache::thrift::TBase {
public:
- DateType(const DateType&);
- DateType& operator=(const DateType&);
- DateType() {
+ DateType(const DateType&) noexcept;
+ DateType& operator=(const DateType&) noexcept;
+ DateType() noexcept {
}
virtual ~DateType() noexcept;
@@ -518,8 +745,8 @@ class DateType : public virtual ::apache::thrift::TBase {
bool operator < (const DateType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -529,12 +756,19 @@ void swap(DateType &a, DateType &b);
std::ostream& operator<<(std::ostream& out, const DateType& obj);
+/**
+ * Logical type to annotate a column that is always null.
+ *
+ * Sometimes when discovering the schema of existing data, values are always
+ * null and the physical type can't be determined. This annotation signals
+ * the case where the physical type was guessed from all null values.
+ */
class NullType : public virtual ::apache::thrift::TBase {
public:
- NullType(const NullType&);
- NullType& operator=(const NullType&);
- NullType() {
+ NullType(const NullType&) noexcept;
+ NullType& operator=(const NullType&) noexcept;
+ NullType() noexcept {
}
virtual ~NullType() noexcept;
@@ -549,8 +783,8 @@ class NullType : public virtual ::apache::thrift::TBase {
bool operator < (const NullType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -560,12 +794,22 @@ void swap(NullType &a, NullType &b);
std::ostream& operator<<(std::ostream& out, const NullType& obj);
+/**
+ * Decimal logical type annotation
+ *
+ * To maintain forward-compatibility in v1, implementations using this logical
+ * type must also set scale and precision on the annotated SchemaElement.
+ *
+ * Allowed for physical types: INT32, INT64, FIXED, and BINARY
+ */
class DecimalType : public virtual ::apache::thrift::TBase {
public:
- DecimalType(const DecimalType&);
- DecimalType& operator=(const DecimalType&);
- DecimalType() : scale(0), precision(0) {
+ DecimalType(const DecimalType&) noexcept;
+ DecimalType& operator=(const DecimalType&) noexcept;
+ DecimalType() noexcept
+ : scale(0),
+ precision(0) {
}
virtual ~DecimalType() noexcept;
@@ -590,8 +834,8 @@ class DecimalType : public virtual ::apache::thrift::TBase {
bool operator < (const DecimalType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -601,12 +845,15 @@ void swap(DecimalType &a, DecimalType &b);
std::ostream& operator<<(std::ostream& out, const DecimalType& obj);
+/**
+ * Time units for logical types
+ */
class MilliSeconds : public virtual ::apache::thrift::TBase {
public:
- MilliSeconds(const MilliSeconds&);
- MilliSeconds& operator=(const MilliSeconds&);
- MilliSeconds() {
+ MilliSeconds(const MilliSeconds&) noexcept;
+ MilliSeconds& operator=(const MilliSeconds&) noexcept;
+ MilliSeconds() noexcept {
}
virtual ~MilliSeconds() noexcept;
@@ -621,8 +868,8 @@ class MilliSeconds : public virtual ::apache::thrift::TBase
{
bool operator < (const MilliSeconds & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -635,9 +882,9 @@ std::ostream& operator<<(std::ostream& out, const
MilliSeconds& obj);
class MicroSeconds : public virtual ::apache::thrift::TBase {
public:
- MicroSeconds(const MicroSeconds&);
- MicroSeconds& operator=(const MicroSeconds&);
- MicroSeconds() {
+ MicroSeconds(const MicroSeconds&) noexcept;
+ MicroSeconds& operator=(const MicroSeconds&) noexcept;
+ MicroSeconds() noexcept {
}
virtual ~MicroSeconds() noexcept;
@@ -652,8 +899,8 @@ class MicroSeconds : public virtual ::apache::thrift::TBase
{
bool operator < (const MicroSeconds & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -666,9 +913,9 @@ std::ostream& operator<<(std::ostream& out, const
MicroSeconds& obj);
class NanoSeconds : public virtual ::apache::thrift::TBase {
public:
- NanoSeconds(const NanoSeconds&);
- NanoSeconds& operator=(const NanoSeconds&);
- NanoSeconds() {
+ NanoSeconds(const NanoSeconds&) noexcept;
+ NanoSeconds& operator=(const NanoSeconds&) noexcept;
+ NanoSeconds() noexcept {
}
virtual ~NanoSeconds() noexcept;
@@ -683,8 +930,8 @@ class NanoSeconds : public virtual ::apache::thrift::TBase {
bool operator < (const NanoSeconds & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -703,9 +950,9 @@ typedef struct _TimeUnit__isset {
class TimeUnit : public virtual ::apache::thrift::TBase {
public:
- TimeUnit(const TimeUnit&);
- TimeUnit& operator=(const TimeUnit&);
- TimeUnit() {
+ TimeUnit(const TimeUnit&) noexcept;
+ TimeUnit& operator=(const TimeUnit&) noexcept;
+ TimeUnit() noexcept {
}
virtual ~TimeUnit() noexcept;
@@ -743,8 +990,8 @@ class TimeUnit : public virtual ::apache::thrift::TBase {
bool operator < (const TimeUnit & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -754,12 +1001,18 @@ void swap(TimeUnit &a, TimeUnit &b);
std::ostream& operator<<(std::ostream& out, const TimeUnit& obj);
+/**
+ * Timestamp logical type annotation
+ *
+ * Allowed for physical types: INT64
+ */
class TimestampType : public virtual ::apache::thrift::TBase {
public:
- TimestampType(const TimestampType&);
- TimestampType& operator=(const TimestampType&);
- TimestampType() : isAdjustedToUTC(0) {
+ TimestampType(const TimestampType&) noexcept;
+ TimestampType& operator=(const TimestampType&) noexcept;
+ TimestampType() noexcept
+ : isAdjustedToUTC(0) {
}
virtual ~TimestampType() noexcept;
@@ -784,8 +1037,8 @@ class TimestampType : public virtual
::apache::thrift::TBase {
bool operator < (const TimestampType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -795,12 +1048,18 @@ void swap(TimestampType &a, TimestampType &b);
std::ostream& operator<<(std::ostream& out, const TimestampType& obj);
+/**
+ * Time logical type annotation
+ *
+ * Allowed for physical types: INT32 (millis), INT64 (micros, nanos)
+ */
class TimeType : public virtual ::apache::thrift::TBase {
public:
- TimeType(const TimeType&);
- TimeType& operator=(const TimeType&);
- TimeType() : isAdjustedToUTC(0) {
+ TimeType(const TimeType&) noexcept;
+ TimeType& operator=(const TimeType&) noexcept;
+ TimeType() noexcept
+ : isAdjustedToUTC(0) {
}
virtual ~TimeType() noexcept;
@@ -825,8 +1084,8 @@ class TimeType : public virtual ::apache::thrift::TBase {
bool operator < (const TimeType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -836,12 +1095,21 @@ void swap(TimeType &a, TimeType &b);
std::ostream& operator<<(std::ostream& out, const TimeType& obj);
+/**
+ * Integer logical type annotation
+ *
+ * bitWidth must be 8, 16, 32, or 64.
+ *
+ * Allowed for physical types: INT32, INT64
+ */
class IntType : public virtual ::apache::thrift::TBase {
public:
- IntType(const IntType&);
- IntType& operator=(const IntType&);
- IntType() : bitWidth(0), isSigned(0) {
+ IntType(const IntType&) noexcept;
+ IntType& operator=(const IntType&) noexcept;
+ IntType() noexcept
+ : bitWidth(0),
+ isSigned(0) {
}
virtual ~IntType() noexcept;
@@ -866,8 +1134,8 @@ class IntType : public virtual ::apache::thrift::TBase {
bool operator < (const IntType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -877,12 +1145,17 @@ void swap(IntType &a, IntType &b);
std::ostream& operator<<(std::ostream& out, const IntType& obj);
+/**
+ * Embedded JSON logical type annotation
+ *
+ * Allowed for physical types: BINARY
+ */
class JsonType : public virtual ::apache::thrift::TBase {
public:
- JsonType(const JsonType&);
- JsonType& operator=(const JsonType&);
- JsonType() {
+ JsonType(const JsonType&) noexcept;
+ JsonType& operator=(const JsonType&) noexcept;
+ JsonType() noexcept {
}
virtual ~JsonType() noexcept;
@@ -897,8 +1170,8 @@ class JsonType : public virtual ::apache::thrift::TBase {
bool operator < (const JsonType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -908,12 +1181,17 @@ void swap(JsonType &a, JsonType &b);
std::ostream& operator<<(std::ostream& out, const JsonType& obj);
+/**
+ * Embedded BSON logical type annotation
+ *
+ * Allowed for physical types: BINARY
+ */
class BsonType : public virtual ::apache::thrift::TBase {
public:
- BsonType(const BsonType&);
- BsonType& operator=(const BsonType&);
- BsonType() {
+ BsonType(const BsonType&) noexcept;
+ BsonType& operator=(const BsonType&) noexcept;
+ BsonType() noexcept {
}
virtual ~BsonType() noexcept;
@@ -928,8 +1206,8 @@ class BsonType : public virtual ::apache::thrift::TBase {
bool operator < (const BsonType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -955,12 +1233,19 @@ typedef struct _LogicalType__isset {
bool UUID :1;
} _LogicalType__isset;
+/**
+ * LogicalType annotations to replace ConvertedType.
+ *
+ * To maintain compatibility, implementations using LogicalType for a
+ * SchemaElement must also set the corresponding ConvertedType (if any)
+ * from the following table.
+ */
class LogicalType : public virtual ::apache::thrift::TBase {
public:
- LogicalType(const LogicalType&);
- LogicalType& operator=(const LogicalType&);
- LogicalType() {
+ LogicalType(const LogicalType&) noexcept;
+ LogicalType& operator=(const LogicalType&) noexcept;
+ LogicalType() noexcept {
}
virtual ~LogicalType() noexcept;
@@ -1068,8 +1353,8 @@ class LogicalType : public virtual
::apache::thrift::TBase {
bool operator < (const LogicalType & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1091,24 +1376,89 @@ typedef struct _SchemaElement__isset {
bool logicalType :1;
} _SchemaElement__isset;
+/**
+ * Represents a element inside a schema definition.
+ * - if it is a group (inner node) then type is undefined and num_children is
defined
+ * - if it is a primitive type (leaf) then type is defined and num_children
is undefined
+ * the nodes are listed in depth first traversal order.
+ */
class SchemaElement : public virtual ::apache::thrift::TBase {
public:
SchemaElement(const SchemaElement&);
SchemaElement& operator=(const SchemaElement&);
- SchemaElement() : type((Type::type)0), type_length(0),
repetition_type((FieldRepetitionType::type)0), name(), num_children(0),
converted_type((ConvertedType::type)0), scale(0), precision(0), field_id(0) {
+ SchemaElement() noexcept
+ : type(static_cast<Type::type>(0)),
+ type_length(0),
+ repetition_type(static_cast<FieldRepetitionType::type>(0)),
+ name(),
+ num_children(0),
+ converted_type(static_cast<ConvertedType::type>(0)),
+ scale(0),
+ precision(0),
+ field_id(0) {
}
virtual ~SchemaElement() noexcept;
+ /**
+ * Data type for this field. Not set if the current element is a non-leaf
node
+ *
+ * @see Type
+ */
Type::type type;
+ /**
+ * If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values.
+ * Otherwise, if specified, this is the maximum bit length to store any of
the values.
+ * (e.g. a low cardinality INT col could have this set to 3). Note that
this is
+ * in the schema, and therefore fixed for the entire file.
+ */
int32_t type_length;
+ /**
+ * repetition of the field. The root of the schema does not have a
repetition_type.
+ * All other nodes must have one
+ *
+ * @see FieldRepetitionType
+ */
FieldRepetitionType::type repetition_type;
+ /**
+ * Name of the field in the schema
+ */
std::string name;
+ /**
+ * Nested fields. Since thrift does not support nested fields,
+ * the nesting is flattened to a single list by a depth-first traversal.
+ * The children count is used to construct the nested relationship.
+ * This field is not set when the element is a primitive type
+ */
int32_t num_children;
+ /**
+ * DEPRECATED: When the schema is the result of a conversion from another
model.
+ * Used to record the original type to help with cross conversion.
+ *
+ * This is superseded by logicalType.
+ *
+ * @see ConvertedType
+ */
ConvertedType::type converted_type;
+ /**
+ * DEPRECATED: Used when this column contains decimal data.
+ * See the DECIMAL converted type for more details.
+ *
+ * This is superseded by using the DecimalType annotation in logicalType.
+ */
int32_t scale;
int32_t precision;
+ /**
+ * When the original schema supports field ids, this will save the
+ * original field id in the parquet schema
+ */
int32_t field_id;
+ /**
+ * The logical type of this SchemaElement
+ *
+ * LogicalType replaces ConvertedType, but ConvertedType is still required
+ * for some logical types to ensure forward-compatibility in format v1.
+ */
LogicalType logicalType;
_SchemaElement__isset __isset;
@@ -1181,8 +1531,8 @@ class SchemaElement : public virtual
::apache::thrift::TBase {
bool operator < (const SchemaElement & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1196,19 +1546,47 @@ typedef struct _DataPageHeader__isset {
bool statistics :1;
} _DataPageHeader__isset;
+/**
+ * Data page header
+ */
class DataPageHeader : public virtual ::apache::thrift::TBase {
public:
DataPageHeader(const DataPageHeader&);
DataPageHeader& operator=(const DataPageHeader&);
- DataPageHeader() : num_values(0), encoding((Encoding::type)0),
definition_level_encoding((Encoding::type)0),
repetition_level_encoding((Encoding::type)0) {
+ DataPageHeader() noexcept
+ : num_values(0),
+ encoding(static_cast<Encoding::type>(0)),
+ definition_level_encoding(static_cast<Encoding::type>(0)),
+ repetition_level_encoding(static_cast<Encoding::type>(0)) {
}
virtual ~DataPageHeader() noexcept;
+ /**
+ * Number of values, including NULLs, in this data page. *
+ */
int32_t num_values;
+ /**
+ * Encoding used for this data page *
+ *
+ * @see Encoding
+ */
Encoding::type encoding;
+ /**
+ * Encoding used for definition levels *
+ *
+ * @see Encoding
+ */
Encoding::type definition_level_encoding;
+ /**
+ * Encoding used for repetition levels *
+ *
+ * @see Encoding
+ */
Encoding::type repetition_level_encoding;
+ /**
+ * Optional statistics for the data in this page*
+ */
Statistics statistics;
_DataPageHeader__isset __isset;
@@ -1245,8 +1623,8 @@ class DataPageHeader : public virtual
::apache::thrift::TBase {
bool operator < (const DataPageHeader & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1259,9 +1637,9 @@ std::ostream& operator<<(std::ostream& out, const
DataPageHeader& obj);
class IndexPageHeader : public virtual ::apache::thrift::TBase {
public:
- IndexPageHeader(const IndexPageHeader&);
- IndexPageHeader& operator=(const IndexPageHeader&);
- IndexPageHeader() {
+ IndexPageHeader(const IndexPageHeader&) noexcept;
+ IndexPageHeader& operator=(const IndexPageHeader&) noexcept;
+ IndexPageHeader() noexcept {
}
virtual ~IndexPageHeader() noexcept;
@@ -1276,8 +1654,8 @@ class IndexPageHeader : public virtual
::apache::thrift::TBase {
bool operator < (const IndexPageHeader & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1291,17 +1669,37 @@ typedef struct _DictionaryPageHeader__isset {
bool is_sorted :1;
} _DictionaryPageHeader__isset;
+/**
+ * The dictionary page must be placed at the first position of the column chunk
+ * if it is partly or completely dictionary encoded. At most one dictionary
page
+ * can be placed in a column chunk.
+ *
+ */
class DictionaryPageHeader : public virtual ::apache::thrift::TBase {
public:
- DictionaryPageHeader(const DictionaryPageHeader&);
- DictionaryPageHeader& operator=(const DictionaryPageHeader&);
- DictionaryPageHeader() : num_values(0), encoding((Encoding::type)0),
is_sorted(0) {
+ DictionaryPageHeader(const DictionaryPageHeader&) noexcept;
+ DictionaryPageHeader& operator=(const DictionaryPageHeader&) noexcept;
+ DictionaryPageHeader() noexcept
+ : num_values(0),
+ encoding(static_cast<Encoding::type>(0)),
+ is_sorted(0) {
}
virtual ~DictionaryPageHeader() noexcept;
+ /**
+ * Number of values in the dictionary *
+ */
int32_t num_values;
+ /**
+ * Encoding using this dictionary page *
+ *
+ * @see Encoding
+ */
Encoding::type encoding;
+ /**
+ * If true, the entries in the dictionary are sorted in ascending order *
+ */
bool is_sorted;
_DictionaryPageHeader__isset __isset;
@@ -1330,8 +1728,8 @@ class DictionaryPageHeader : public virtual
::apache::thrift::TBase {
bool operator < (const DictionaryPageHeader & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1346,22 +1744,66 @@ typedef struct _DataPageHeaderV2__isset {
bool statistics :1;
} _DataPageHeaderV2__isset;
+/**
+ * New page format allowing reading levels without decompressing the data
+ * Repetition and definition levels are uncompressed
+ * The remaining section containing the data is compressed if is_compressed is
true
+ *
+ */
class DataPageHeaderV2 : public virtual ::apache::thrift::TBase {
public:
DataPageHeaderV2(const DataPageHeaderV2&);
DataPageHeaderV2& operator=(const DataPageHeaderV2&);
- DataPageHeaderV2() : num_values(0), num_nulls(0), num_rows(0),
encoding((Encoding::type)0), definition_levels_byte_length(0),
repetition_levels_byte_length(0), is_compressed(true) {
+ DataPageHeaderV2() noexcept
+ : num_values(0),
+ num_nulls(0),
+ num_rows(0),
+ encoding(static_cast<Encoding::type>(0)),
+ definition_levels_byte_length(0),
+ repetition_levels_byte_length(0),
+ is_compressed(true) {
}
virtual ~DataPageHeaderV2() noexcept;
+ /**
+ * Number of values, including NULLs, in this data page. *
+ */
int32_t num_values;
+ /**
+ * Number of NULL values, in this data page.
+ * Number of non-null = num_values - num_nulls which is also the number of
values in the data section *
+ */
int32_t num_nulls;
+ /**
+ * Number of rows in this data page. which means pages change on record
boundaries (r = 0) *
+ */
int32_t num_rows;
+ /**
+ * Encoding used for data in this page *
+ *
+ * @see Encoding
+ */
Encoding::type encoding;
+ /**
+ * length of the definition levels
+ */
int32_t definition_levels_byte_length;
+ /**
+ * length of the repetition levels
+ */
int32_t repetition_levels_byte_length;
+ /**
+ * whether the values are compressed.
+ * Which means the section of the page between
+ * definition_levels_byte_length + repetition_levels_byte_length + 1 and
compressed_page_size (included)
+ * is compressed with the compression_codec.
+ * If missing it is considered compressed
+ */
bool is_compressed;
+ /**
+ * optional statistics for the data in this page *
+ */
Statistics statistics;
_DataPageHeaderV2__isset __isset;
@@ -1412,8 +1854,8 @@ class DataPageHeaderV2 : public virtual
::apache::thrift::TBase {
bool operator < (const DataPageHeaderV2 & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1423,12 +1865,15 @@ void swap(DataPageHeaderV2 &a, DataPageHeaderV2 &b);
std::ostream& operator<<(std::ostream& out, const DataPageHeaderV2& obj);
+/**
+ * Block-based algorithm type annotation. *
+ */
class SplitBlockAlgorithm : public virtual ::apache::thrift::TBase {
public:
- SplitBlockAlgorithm(const SplitBlockAlgorithm&);
- SplitBlockAlgorithm& operator=(const SplitBlockAlgorithm&);
- SplitBlockAlgorithm() {
+ SplitBlockAlgorithm(const SplitBlockAlgorithm&) noexcept;
+ SplitBlockAlgorithm& operator=(const SplitBlockAlgorithm&) noexcept;
+ SplitBlockAlgorithm() noexcept {
}
virtual ~SplitBlockAlgorithm() noexcept;
@@ -1443,8 +1888,8 @@ class SplitBlockAlgorithm : public virtual
::apache::thrift::TBase {
bool operator < (const SplitBlockAlgorithm & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1458,15 +1903,21 @@ typedef struct _BloomFilterAlgorithm__isset {
bool BLOCK :1;
} _BloomFilterAlgorithm__isset;
+/**
+ * The algorithm used in Bloom filter. *
+ */
class BloomFilterAlgorithm : public virtual ::apache::thrift::TBase {
public:
- BloomFilterAlgorithm(const BloomFilterAlgorithm&);
- BloomFilterAlgorithm& operator=(const BloomFilterAlgorithm&);
- BloomFilterAlgorithm() {
+ BloomFilterAlgorithm(const BloomFilterAlgorithm&) noexcept;
+ BloomFilterAlgorithm& operator=(const BloomFilterAlgorithm&) noexcept;
+ BloomFilterAlgorithm() noexcept {
}
virtual ~BloomFilterAlgorithm() noexcept;
+ /**
+ * Block-based Bloom filter. *
+ */
SplitBlockAlgorithm BLOCK;
_BloomFilterAlgorithm__isset __isset;
@@ -1487,8 +1938,8 @@ class BloomFilterAlgorithm : public virtual
::apache::thrift::TBase {
bool operator < (const BloomFilterAlgorithm & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1498,12 +1949,17 @@ void swap(BloomFilterAlgorithm &a, BloomFilterAlgorithm
&b);
std::ostream& operator<<(std::ostream& out, const BloomFilterAlgorithm& obj);
+/**
+ * Hash strategy type annotation. xxHash is an extremely fast
non-cryptographic hash
+ * algorithm. It uses 64 bits version of xxHash.
+ *
+ */
class XxHash : public virtual ::apache::thrift::TBase {
public:
- XxHash(const XxHash&);
- XxHash& operator=(const XxHash&);
- XxHash() {
+ XxHash(const XxHash&) noexcept;
+ XxHash& operator=(const XxHash&) noexcept;
+ XxHash() noexcept {
}
virtual ~XxHash() noexcept;
@@ -1518,8 +1974,8 @@ class XxHash : public virtual ::apache::thrift::TBase {
bool operator < (const XxHash & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1533,15 +1989,23 @@ typedef struct _BloomFilterHash__isset {
bool XXHASH :1;
} _BloomFilterHash__isset;
+/**
+ * The hash function used in Bloom filter. This function takes the hash of a
column value
+ * using plain encoding.
+ *
+ */
class BloomFilterHash : public virtual ::apache::thrift::TBase {
public:
- BloomFilterHash(const BloomFilterHash&);
- BloomFilterHash& operator=(const BloomFilterHash&);
- BloomFilterHash() {
+ BloomFilterHash(const BloomFilterHash&) noexcept;
+ BloomFilterHash& operator=(const BloomFilterHash&) noexcept;
+ BloomFilterHash() noexcept {
}
virtual ~BloomFilterHash() noexcept;
+ /**
+ * xxHash Strategy. *
+ */
XxHash XXHASH;
_BloomFilterHash__isset __isset;
@@ -1562,8 +2026,8 @@ class BloomFilterHash : public virtual
::apache::thrift::TBase {
bool operator < (const BloomFilterHash & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1573,12 +2037,16 @@ void swap(BloomFilterHash &a, BloomFilterHash &b);
std::ostream& operator<<(std::ostream& out, const BloomFilterHash& obj);
+/**
+ * The compression used in the Bloom filter.
+ *
+ */
class Uncompressed : public virtual ::apache::thrift::TBase {
public:
- Uncompressed(const Uncompressed&);
- Uncompressed& operator=(const Uncompressed&);
- Uncompressed() {
+ Uncompressed(const Uncompressed&) noexcept;
+ Uncompressed& operator=(const Uncompressed&) noexcept;
+ Uncompressed() noexcept {
}
virtual ~Uncompressed() noexcept;
@@ -1593,8 +2061,8 @@ class Uncompressed : public virtual
::apache::thrift::TBase {
bool operator < (const Uncompressed & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1611,9 +2079,9 @@ typedef struct _BloomFilterCompression__isset {
class BloomFilterCompression : public virtual ::apache::thrift::TBase {
public:
- BloomFilterCompression(const BloomFilterCompression&);
- BloomFilterCompression& operator=(const BloomFilterCompression&);
- BloomFilterCompression() {
+ BloomFilterCompression(const BloomFilterCompression&) noexcept;
+ BloomFilterCompression& operator=(const BloomFilterCompression&) noexcept;
+ BloomFilterCompression() noexcept {
}
virtual ~BloomFilterCompression() noexcept;
@@ -1637,8 +2105,8 @@ class BloomFilterCompression : public virtual
::apache::thrift::TBase {
bool operator < (const BloomFilterCompression & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1648,18 +2116,36 @@ void swap(BloomFilterCompression &a,
BloomFilterCompression &b);
std::ostream& operator<<(std::ostream& out, const BloomFilterCompression& obj);
+/**
+ * Bloom filter header is stored at beginning of Bloom filter data of each
column
+ * and followed by its bitset.
+ *
+ */
class BloomFilterHeader : public virtual ::apache::thrift::TBase {
public:
- BloomFilterHeader(const BloomFilterHeader&);
- BloomFilterHeader& operator=(const BloomFilterHeader&);
- BloomFilterHeader() : numBytes(0) {
+ BloomFilterHeader(const BloomFilterHeader&) noexcept;
+ BloomFilterHeader& operator=(const BloomFilterHeader&) noexcept;
+ BloomFilterHeader() noexcept
+ : numBytes(0) {
}
virtual ~BloomFilterHeader() noexcept;
+ /**
+ * The size of bitset in bytes *
+ */
int32_t numBytes;
+ /**
+ * The algorithm for setting bits. *
+ */
BloomFilterAlgorithm algorithm;
+ /**
+ * The hash function used for Bloom filter. *
+ */
BloomFilterHash hash;
+ /**
+ * The compression used in the Bloom filter *
+ */
BloomFilterCompression compression;
void __set_numBytes(const int32_t val);
@@ -1688,8 +2174,8 @@ class BloomFilterHeader : public virtual
::apache::thrift::TBase {
bool operator < (const BloomFilterHeader & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1712,13 +2198,46 @@ class PageHeader : public virtual
::apache::thrift::TBase {
PageHeader(const PageHeader&);
PageHeader& operator=(const PageHeader&);
- PageHeader() : type((PageType::type)0), uncompressed_page_size(0),
compressed_page_size(0), crc(0) {
+ PageHeader() noexcept
+ : type(static_cast<PageType::type>(0)),
+ uncompressed_page_size(0),
+ compressed_page_size(0),
+ crc(0) {
}
virtual ~PageHeader() noexcept;
+ /**
+ * the type of the page: indicates which of the *_header fields is set *
+ *
+ * @see PageType
+ */
PageType::type type;
+ /**
+ * Uncompressed page size in bytes (not including this header) *
+ */
int32_t uncompressed_page_size;
+ /**
+ * Compressed (and potentially encrypted) page size in bytes, not including
this header *
+ */
int32_t compressed_page_size;
+ /**
+ * The 32-bit CRC checksum for the page, to be be calculated as follows:
+ *
+ * - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7,
+ * the same as in e.g. GZip).
+ * - All page types can have a CRC (v1 and v2 data pages, dictionary pages,
+ * etc.).
+ * - The CRC is computed on the serialization binary representation of the
page
+ * (as written to disk), excluding the page header. For example, for v1
+ * data pages, the CRC is computed on the concatenation of repetition
levels,
+ * definition levels and column values (optionally compressed, optionally
+ * encrypted).
+ * - The CRC computation therefore takes place after any compression
+ * and encryption steps, if any.
+ *
+ * If enabled, this allows for disabling checksumming in HDFS if only a few
+ * pages need to be read.
+ */
int32_t crc;
DataPageHeader data_page_header;
IndexPageHeader index_page_header;
@@ -1779,8 +2298,8 @@ class PageHeader : public virtual ::apache::thrift::TBase
{
bool operator < (const PageHeader & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1794,12 +2313,17 @@ typedef struct _KeyValue__isset {
bool value :1;
} _KeyValue__isset;
+/**
+ * Wrapper struct to store key values
+ */
class KeyValue : public virtual ::apache::thrift::TBase {
public:
KeyValue(const KeyValue&);
KeyValue& operator=(const KeyValue&);
- KeyValue() : key(), value() {
+ KeyValue() noexcept
+ : key(),
+ value() {
}
virtual ~KeyValue() noexcept;
@@ -1828,8 +2352,8 @@ class KeyValue : public virtual ::apache::thrift::TBase {
bool operator < (const KeyValue & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1839,17 +2363,33 @@ void swap(KeyValue &a, KeyValue &b);
std::ostream& operator<<(std::ostream& out, const KeyValue& obj);
+/**
+ * Wrapper struct to specify sort order
+ */
class SortingColumn : public virtual ::apache::thrift::TBase {
public:
- SortingColumn(const SortingColumn&);
- SortingColumn& operator=(const SortingColumn&);
- SortingColumn() : column_idx(0), descending(0), nulls_first(0) {
+ SortingColumn(const SortingColumn&) noexcept;
+ SortingColumn& operator=(const SortingColumn&) noexcept;
+ SortingColumn() noexcept
+ : column_idx(0),
+ descending(0),
+ nulls_first(0) {
}
virtual ~SortingColumn() noexcept;
+ /**
+ * The column index (in this row group) *
+ */
int32_t column_idx;
+ /**
+ * If true, indicates this column is sorted in descending order. *
+ */
bool descending;
+ /**
+ * If true, nulls will come before non-null values, otherwise,
+ * nulls go at the end.
+ */
bool nulls_first;
void __set_column_idx(const int32_t val);
@@ -1874,8 +2414,8 @@ class SortingColumn : public virtual
::apache::thrift::TBase {
bool operator < (const SortingColumn & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1885,17 +2425,36 @@ void swap(SortingColumn &a, SortingColumn &b);
std::ostream& operator<<(std::ostream& out, const SortingColumn& obj);
+/**
+ * statistics of a given page type and encoding
+ */
class PageEncodingStats : public virtual ::apache::thrift::TBase {
public:
- PageEncodingStats(const PageEncodingStats&);
- PageEncodingStats& operator=(const PageEncodingStats&);
- PageEncodingStats() : page_type((PageType::type)0),
encoding((Encoding::type)0), count(0) {
+ PageEncodingStats(const PageEncodingStats&) noexcept;
+ PageEncodingStats& operator=(const PageEncodingStats&) noexcept;
+ PageEncodingStats() noexcept
+ : page_type(static_cast<PageType::type>(0)),
+ encoding(static_cast<Encoding::type>(0)),
+ count(0) {
}
virtual ~PageEncodingStats() noexcept;
+ /**
+ * the page type (data/dic/...) *
+ *
+ * @see PageType
+ */
PageType::type page_type;
+ /**
+ * encoding of the page *
+ *
+ * @see Encoding
+ */
Encoding::type encoding;
+ /**
+ * number of pages of this type with this encoding *
+ */
int32_t count;
void __set_page_type(const PageType::type val);
@@ -1920,8 +2479,8 @@ class PageEncodingStats : public virtual
::apache::thrift::TBase {
bool operator < (const PageEncodingStats & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -1940,28 +2499,90 @@ typedef struct _ColumnMetaData__isset {
bool bloom_filter_offset :1;
} _ColumnMetaData__isset;
+/**
+ * Description for column metadata
+ */
class ColumnMetaData : public virtual ::apache::thrift::TBase {
public:
ColumnMetaData(const ColumnMetaData&);
ColumnMetaData& operator=(const ColumnMetaData&);
- ColumnMetaData() : type((Type::type)0), codec((CompressionCodec::type)0),
num_values(0), total_uncompressed_size(0), total_compressed_size(0),
data_page_offset(0), index_page_offset(0), dictionary_page_offset(0),
bloom_filter_offset(0) {
+ ColumnMetaData() noexcept
+ : type(static_cast<Type::type>(0)),
+ codec(static_cast<CompressionCodec::type>(0)),
+ num_values(0),
+ total_uncompressed_size(0),
+ total_compressed_size(0),
+ data_page_offset(0),
+ index_page_offset(0),
+ dictionary_page_offset(0),
+ bloom_filter_offset(0) {
}
virtual ~ColumnMetaData() noexcept;
+ /**
+ * Type of this column *
+ *
+ * @see Type
+ */
Type::type type;
+ /**
+ * Set of all encodings used for this column. The purpose is to validate
+ * whether we can decode those pages. *
+ */
std::vector<Encoding::type> encodings;
+ /**
+ * Path in schema *
+ */
std::vector<std::string> path_in_schema;
+ /**
+ * Compression codec *
+ *
+ * @see CompressionCodec
+ */
CompressionCodec::type codec;
+ /**
+ * Number of values in this column *
+ */
int64_t num_values;
+ /**
+ * total byte size of all uncompressed pages in this column chunk (including
the headers) *
+ */
int64_t total_uncompressed_size;
+ /**
+ * total byte size of all compressed, and potentially encrypted, pages
+ * in this column chunk (including the headers) *
+ */
int64_t total_compressed_size;
+ /**
+ * Optional key/value metadata *
+ */
std::vector<KeyValue> key_value_metadata;
+ /**
+ * Byte offset from beginning of file to first data page *
+ */
int64_t data_page_offset;
+ /**
+ * Byte offset from beginning of file to root index page *
+ */
int64_t index_page_offset;
+ /**
+ * Byte offset from the beginning of file to first (only) dictionary page *
+ */
int64_t dictionary_page_offset;
+ /**
+ * optional statistics for this column chunk
+ */
Statistics statistics;
+ /**
+ * Set of all encodings used for pages in this column chunk.
+ * This information can be used to determine if all data pages are
+ * dictionary encoded for example *
+ */
std::vector<PageEncodingStats> encoding_stats;
+ /**
+ * Byte offset from beginning of file to Bloom filter data. *
+ */
int64_t bloom_filter_offset;
_ColumnMetaData__isset __isset;
@@ -2044,8 +2665,8 @@ class ColumnMetaData : public virtual
::apache::thrift::TBase {
bool operator < (const ColumnMetaData & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2058,9 +2679,9 @@ std::ostream& operator<<(std::ostream& out, const
ColumnMetaData& obj);
class EncryptionWithFooterKey : public virtual ::apache::thrift::TBase {
public:
- EncryptionWithFooterKey(const EncryptionWithFooterKey&);
- EncryptionWithFooterKey& operator=(const EncryptionWithFooterKey&);
- EncryptionWithFooterKey() {
+ EncryptionWithFooterKey(const EncryptionWithFooterKey&) noexcept;
+ EncryptionWithFooterKey& operator=(const EncryptionWithFooterKey&) noexcept;
+ EncryptionWithFooterKey() noexcept {
}
virtual ~EncryptionWithFooterKey() noexcept;
@@ -2075,8 +2696,8 @@ class EncryptionWithFooterKey : public virtual
::apache::thrift::TBase {
bool operator < (const EncryptionWithFooterKey & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2095,11 +2716,18 @@ class EncryptionWithColumnKey : public virtual
::apache::thrift::TBase {
EncryptionWithColumnKey(const EncryptionWithColumnKey&);
EncryptionWithColumnKey& operator=(const EncryptionWithColumnKey&);
- EncryptionWithColumnKey() : key_metadata() {
+ EncryptionWithColumnKey() noexcept
+ : key_metadata() {
}
virtual ~EncryptionWithColumnKey() noexcept;
+ /**
+ * Column path in schema *
+ */
std::vector<std::string> path_in_schema;
+ /**
+ * Retrieval metadata of column encryption key *
+ */
std::string key_metadata;
_EncryptionWithColumnKey__isset __isset;
@@ -2124,8 +2752,8 @@ class EncryptionWithColumnKey : public virtual
::apache::thrift::TBase {
bool operator < (const EncryptionWithColumnKey & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2145,7 +2773,7 @@ class ColumnCryptoMetaData : public virtual
::apache::thrift::TBase {
ColumnCryptoMetaData(const ColumnCryptoMetaData&);
ColumnCryptoMetaData& operator=(const ColumnCryptoMetaData&);
- ColumnCryptoMetaData() {
+ ColumnCryptoMetaData() noexcept {
}
virtual ~ColumnCryptoMetaData() noexcept;
@@ -2176,8 +2804,8 @@ class ColumnCryptoMetaData : public virtual
::apache::thrift::TBase {
bool operator < (const ColumnCryptoMetaData & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2203,18 +2831,57 @@ class ColumnChunk : public virtual
::apache::thrift::TBase {
ColumnChunk(const ColumnChunk&);
ColumnChunk& operator=(const ColumnChunk&);
- ColumnChunk() : file_path(), file_offset(0), offset_index_offset(0),
offset_index_length(0), column_index_offset(0), column_index_length(0),
encrypted_column_metadata() {
+ ColumnChunk() noexcept
+ : file_path(),
+ file_offset(0),
+ offset_index_offset(0),
+ offset_index_length(0),
+ column_index_offset(0),
+ column_index_length(0),
+ encrypted_column_metadata() {
}
virtual ~ColumnChunk() noexcept;
+ /**
+ * File where column data is stored. If not set, assumed to be same file as
+ * metadata. This path is relative to the current file.
+ *
+ */
std::string file_path;
+ /**
+ * Byte offset in file_path to the ColumnMetaData *
+ */
int64_t file_offset;
+ /**
+ * Column metadata for this chunk. This is the same content as what is at
+ * file_path/file_offset. Having it here has it replicated in the file
+ * metadata.
+ *
+ */
ColumnMetaData meta_data;
+ /**
+ * File offset of ColumnChunk's OffsetIndex *
+ */
int64_t offset_index_offset;
+ /**
+ * Size of ColumnChunk's OffsetIndex, in bytes *
+ */
int32_t offset_index_length;
+ /**
+ * File offset of ColumnChunk's ColumnIndex *
+ */
int64_t column_index_offset;
+ /**
+ * Size of ColumnChunk's ColumnIndex, in bytes *
+ */
int32_t column_index_length;
+ /**
+ * Crypto metadata of encrypted columns *
+ */
ColumnCryptoMetaData crypto_metadata;
+ /**
+ * Encrypted column metadata for this chunk *
+ */
std::string encrypted_column_metadata;
_ColumnChunk__isset __isset;
@@ -2281,8 +2948,8 @@ class ColumnChunk : public virtual
::apache::thrift::TBase {
bool operator < (const ColumnChunk & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2304,16 +2971,47 @@ class RowGroup : public virtual ::apache::thrift::TBase
{
RowGroup(const RowGroup&);
RowGroup& operator=(const RowGroup&);
- RowGroup() : total_byte_size(0), num_rows(0), file_offset(0),
total_compressed_size(0), ordinal(0) {
+ RowGroup() noexcept
+ : total_byte_size(0),
+ num_rows(0),
+ file_offset(0),
+ total_compressed_size(0),
+ ordinal(0) {
}
virtual ~RowGroup() noexcept;
+ /**
+ * Metadata for each column chunk in this row group.
+ * This list must have the same order as the SchemaElement list in
FileMetaData.
+ *
+ */
std::vector<ColumnChunk> columns;
+ /**
+ * Total byte size of all the uncompressed column data in this row group *
+ */
int64_t total_byte_size;
+ /**
+ * Number of rows in this row group *
+ */
int64_t num_rows;
+ /**
+ * If set, specifies a sort ordering of the rows in this RowGroup.
+ * The sorting columns can be a subset of all the columns.
+ */
std::vector<SortingColumn> sorting_columns;
+ /**
+ * Byte offset from beginning of file to first page (data or dictionary)
+ * in this row group *
+ */
int64_t file_offset;
+ /**
+ * Total byte size of all compressed (and potentially encrypted) column data
+ * in this row group *
+ */
int64_t total_compressed_size;
+ /**
+ * Row group ordinal in the file *
+ */
int16_t ordinal;
_RowGroup__isset __isset;
@@ -2364,8 +3062,8 @@ class RowGroup : public virtual ::apache::thrift::TBase {
bool operator < (const RowGroup & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2375,12 +3073,15 @@ void swap(RowGroup &a, RowGroup &b);
std::ostream& operator<<(std::ostream& out, const RowGroup& obj);
+/**
+ * Empty struct to signal the order defined by the physical or logical type
+ */
class TypeDefinedOrder : public virtual ::apache::thrift::TBase {
public:
- TypeDefinedOrder(const TypeDefinedOrder&);
- TypeDefinedOrder& operator=(const TypeDefinedOrder&);
- TypeDefinedOrder() {
+ TypeDefinedOrder(const TypeDefinedOrder&) noexcept;
+ TypeDefinedOrder& operator=(const TypeDefinedOrder&) noexcept;
+ TypeDefinedOrder() noexcept {
}
virtual ~TypeDefinedOrder() noexcept;
@@ -2395,8 +3096,8 @@ class TypeDefinedOrder : public virtual
::apache::thrift::TBase {
bool operator < (const TypeDefinedOrder & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2410,15 +3111,77 @@ typedef struct _ColumnOrder__isset {
bool TYPE_ORDER :1;
} _ColumnOrder__isset;
+/**
+ * Union to specify the order used for the min_value and max_value fields for a
+ * column. This union takes the role of an enhanced enum that allows rich
+ * elements (which will be needed for a collation-based ordering in the
future).
+ *
+ * Possible values are:
+ * * TypeDefinedOrder - the column uses the order defined by its logical or
+ * physical type (if there is no logical type).
+ *
+ * If the reader does not support the value of this union, min and max stats
+ * for this column should be ignored.
+ */
class ColumnOrder : public virtual ::apache::thrift::TBase {
public:
- ColumnOrder(const ColumnOrder&);
- ColumnOrder& operator=(const ColumnOrder&);
- ColumnOrder() {
+ ColumnOrder(const ColumnOrder&) noexcept;
+ ColumnOrder& operator=(const ColumnOrder&) noexcept;
+ ColumnOrder() noexcept {
}
virtual ~ColumnOrder() noexcept;
+ /**
+ * The sort orders for logical types are:
+ * UTF8 - unsigned byte-wise comparison
+ * INT8 - signed comparison
+ * INT16 - signed comparison
+ * INT32 - signed comparison
+ * INT64 - signed comparison
+ * UINT8 - unsigned comparison
+ * UINT16 - unsigned comparison
+ * UINT32 - unsigned comparison
+ * UINT64 - unsigned comparison
+ * DECIMAL - signed comparison of the represented value
+ * DATE - signed comparison
+ * TIME_MILLIS - signed comparison
+ * TIME_MICROS - signed comparison
+ * TIMESTAMP_MILLIS - signed comparison
+ * TIMESTAMP_MICROS - signed comparison
+ * INTERVAL - unsigned comparison
+ * JSON - unsigned byte-wise comparison
+ * BSON - unsigned byte-wise comparison
+ * ENUM - unsigned byte-wise comparison
+ * LIST - undefined
+ * MAP - undefined
+ *
+ * In the absence of logical types, the sort order is determined by the
physical type:
+ * BOOLEAN - false, true
+ * INT32 - signed comparison
+ * INT64 - signed comparison
+ * INT96 (only used for legacy timestamps) - undefined
+ * FLOAT - signed comparison of the represented value (*)
+ * DOUBLE - signed comparison of the represented value (*)
+ * BYTE_ARRAY - unsigned byte-wise comparison
+ * FIXED_LEN_BYTE_ARRAY - unsigned byte-wise comparison
+ *
+ * (*) Because the sorting order is not specified properly for floating
+ * point values (relations vs. total ordering) the following
+ * compatibility rules should be applied when reading statistics:
+ * - If the min is a NaN, it should be ignored.
+ * - If the max is a NaN, it should be ignored.
+ * - If the min is +0, the row group may contain -0 values as well.
+ * - If the max is -0, the row group may contain +0 values as well.
+ * - When looking for NaN values, min and max should be ignored.
+ *
+ * When writing statistics the following rules should be followed:
+ * - NaNs should not be written to min or max statistics fields.
+ * - If the computed max value is zero (whether negative or positive),
+ * `+0.0` should be written into the max statistics field.
+ * - If the computed min value is zero (whether negative or positive),
+ * `-0.0` should be written into the min statistics field.
+ */
TypeDefinedOrder TYPE_ORDER;
_ColumnOrder__isset __isset;
@@ -2439,8 +3202,8 @@ class ColumnOrder : public virtual
::apache::thrift::TBase {
bool operator < (const ColumnOrder & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2453,14 +3216,28 @@ std::ostream& operator<<(std::ostream& out, const
ColumnOrder& obj);
class PageLocation : public virtual ::apache::thrift::TBase {
public:
- PageLocation(const PageLocation&);
- PageLocation& operator=(const PageLocation&);
- PageLocation() : offset(0), compressed_page_size(0), first_row_index(0) {
+ PageLocation(const PageLocation&) noexcept;
+ PageLocation& operator=(const PageLocation&) noexcept;
+ PageLocation() noexcept
+ : offset(0),
+ compressed_page_size(0),
+ first_row_index(0) {
}
virtual ~PageLocation() noexcept;
+ /**
+ * Offset of the page in the file *
+ */
int64_t offset;
+ /**
+ * Size of the page, including header. Sum of compressed_page_size and header
+ * length
+ */
int32_t compressed_page_size;
+ /**
+ * Index within the RowGroup of the first row of the page; this means pages
+ * change on record boundaries (r = 0).
+ */
int64_t first_row_index;
void __set_offset(const int64_t val);
@@ -2485,8 +3262,8 @@ class PageLocation : public virtual
::apache::thrift::TBase {
bool operator < (const PageLocation & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2501,10 +3278,14 @@ class OffsetIndex : public virtual
::apache::thrift::TBase {
OffsetIndex(const OffsetIndex&);
OffsetIndex& operator=(const OffsetIndex&);
- OffsetIndex() {
+ OffsetIndex() noexcept {
}
virtual ~OffsetIndex() noexcept;
+ /**
+ * PageLocations, ordered by increasing PageLocation.offset. It is required
+ * that page_locations[i].first_row_index <
page_locations[i+1].first_row_index.
+ */
std::vector<PageLocation> page_locations;
void __set_page_locations(const std::vector<PageLocation> & val);
@@ -2521,8 +3302,8 @@ class OffsetIndex : public virtual
::apache::thrift::TBase {
bool operator < (const OffsetIndex & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2536,19 +3317,52 @@ typedef struct _ColumnIndex__isset {
bool null_counts :1;
} _ColumnIndex__isset;
+/**
+ * Description for ColumnIndex.
+ * Each <array-field>[i] refers to the page at OffsetIndex.page_locations[i]
+ */
class ColumnIndex : public virtual ::apache::thrift::TBase {
public:
ColumnIndex(const ColumnIndex&);
ColumnIndex& operator=(const ColumnIndex&);
- ColumnIndex() : boundary_order((BoundaryOrder::type)0) {
+ ColumnIndex() noexcept
+ : boundary_order(static_cast<BoundaryOrder::type>(0)) {
}
virtual ~ColumnIndex() noexcept;
+ /**
+ * A list of Boolean values to determine the validity of the corresponding
+ * min and max values. If true, a page contains only null values, and writers
+ * have to set the corresponding entries in min_values and max_values to
+ * byte[0], so that all lists have the same length. If false, the
+ * corresponding entries in min_values and max_values must be valid.
+ */
std::vector<bool> null_pages;
+ /**
+ * Two lists containing lower and upper bounds for the values of each page
+ * determined by the ColumnOrder of the column. These may be the actual
+ * minimum and maximum values found on a page, but can also be (more compact)
+ * values that do not exist on a page. For example, instead of storing
""Blart
+ * Versenwald III", a writer may set min_values[i]="B", max_values[i]="C".
+ * Such more compact values must still be valid values within the column's
+ * logical type. Readers must make sure that list entries are populated
before
+ * using them by inspecting null_pages.
+ */
std::vector<std::string> min_values;
std::vector<std::string> max_values;
+ /**
+ * Stores whether both min_values and max_values are ordered and if so, in
+ * which direction. This allows readers to perform binary searches in both
+ * lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
+ * if the lists are ordered.
+ *
+ * @see BoundaryOrder
+ */
BoundaryOrder::type boundary_order;
+ /**
+ * A list containing the number of null values for each page *
+ */
std::vector<int64_t> null_counts;
_ColumnIndex__isset __isset;
@@ -2585,8 +3399,8 @@ class ColumnIndex : public virtual
::apache::thrift::TBase {
bool operator < (const ColumnIndex & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2607,12 +3421,25 @@ class AesGcmV1 : public virtual ::apache::thrift::TBase
{
AesGcmV1(const AesGcmV1&);
AesGcmV1& operator=(const AesGcmV1&);
- AesGcmV1() : aad_prefix(), aad_file_unique(), supply_aad_prefix(0) {
+ AesGcmV1() noexcept
+ : aad_prefix(),
+ aad_file_unique(),
+ supply_aad_prefix(0) {
}
virtual ~AesGcmV1() noexcept;
+ /**
+ * AAD prefix *
+ */
std::string aad_prefix;
+ /**
+ * Unique file identifier part of AAD suffix *
+ */
std::string aad_file_unique;
+ /**
+ * In files encrypted with AAD prefix without storing it,
+ * readers must supply the prefix *
+ */
bool supply_aad_prefix;
_AesGcmV1__isset __isset;
@@ -2645,8 +3472,8 @@ class AesGcmV1 : public virtual ::apache::thrift::TBase {
bool operator < (const AesGcmV1 & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2667,12 +3494,25 @@ class AesGcmCtrV1 : public virtual
::apache::thrift::TBase {
AesGcmCtrV1(const AesGcmCtrV1&);
AesGcmCtrV1& operator=(const AesGcmCtrV1&);
- AesGcmCtrV1() : aad_prefix(), aad_file_unique(), supply_aad_prefix(0) {
+ AesGcmCtrV1() noexcept
+ : aad_prefix(),
+ aad_file_unique(),
+ supply_aad_prefix(0) {
}
virtual ~AesGcmCtrV1() noexcept;
+ /**
+ * AAD prefix *
+ */
std::string aad_prefix;
+ /**
+ * Unique file identifier part of AAD suffix *
+ */
std::string aad_file_unique;
+ /**
+ * In files encrypted with AAD prefix without storing it,
+ * readers must supply the prefix *
+ */
bool supply_aad_prefix;
_AesGcmCtrV1__isset __isset;
@@ -2705,8 +3545,8 @@ class AesGcmCtrV1 : public virtual
::apache::thrift::TBase {
bool operator < (const AesGcmCtrV1 & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2726,7 +3566,7 @@ class EncryptionAlgorithm : public virtual
::apache::thrift::TBase {
EncryptionAlgorithm(const EncryptionAlgorithm&);
EncryptionAlgorithm& operator=(const EncryptionAlgorithm&);
- EncryptionAlgorithm() {
+ EncryptionAlgorithm() noexcept {
}
virtual ~EncryptionAlgorithm() noexcept;
@@ -2757,8 +3597,8 @@ class EncryptionAlgorithm : public virtual
::apache::thrift::TBase {
bool operator < (const EncryptionAlgorithm & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2776,23 +3616,81 @@ typedef struct _FileMetaData__isset {
bool footer_signing_key_metadata :1;
} _FileMetaData__isset;
+/**
+ * Description for file metadata
+ */
class FileMetaData : public virtual ::apache::thrift::TBase {
public:
FileMetaData(const FileMetaData&);
FileMetaData& operator=(const FileMetaData&);
- FileMetaData() : version(0), num_rows(0), created_by(),
footer_signing_key_metadata() {
+ FileMetaData() noexcept
+ : version(0),
+ num_rows(0),
+ created_by(),
+ footer_signing_key_metadata() {
}
virtual ~FileMetaData() noexcept;
+ /**
+ * Version of this file *
+ */
int32_t version;
+ /**
+ * Parquet schema for this file. This schema contains metadata for all the
columns.
+ * The schema is represented as a tree with a single root. The nodes of the
tree
+ * are flattened to a list by doing a depth-first traversal.
+ * The column metadata contains the path in the schema for that column which
can be
+ * used to map columns to nodes in the schema.
+ * The first element is the root *
+ */
std::vector<SchemaElement> schema;
+ /**
+ * Number of rows in this file *
+ */
int64_t num_rows;
+ /**
+ * Row groups in this file *
+ */
std::vector<RowGroup> row_groups;
+ /**
+ * Optional key/value metadata *
+ */
std::vector<KeyValue> key_value_metadata;
+ /**
+ * String for application that wrote this file. This should be in the format
+ * <Application> version <App Version> (build <App Build Hash>).
+ * e.g. impala version 1.0 (build 6cf94d29b2b7115df4de2c06e2ab4326d721eb55)
+ *
+ */
std::string created_by;
+ /**
+ * Sort order used for the min_value and max_value fields in the Statistics
+ * objects and the min_values and max_values fields in the ColumnIndex
+ * objects of each column in this file. Sort orders are listed in the order
+ * matching the columns in the schema. The indexes are not necessary the same
+ * though, because only leaf nodes of the schema are represented in the list
+ * of sort orders.
+ *
+ * Without column_orders, the meaning of the min_value and max_value fields
+ * in the Statistics object and the ColumnIndex object is undefined. To
ensure
+ * well-defined behaviour, if these fields are written to a Parquet file,
+ * column_orders must be written as well.
+ *
+ * The obsolete min and max fields in the Statistics object are always sorted
+ * by signed comparison regardless of column_orders.
+ */
std::vector<ColumnOrder> column_orders;
+ /**
+ * Encryption algorithm. This field is set only in encrypted files
+ * with plaintext footer. Files with encrypted footer store algorithm id
+ * in FileCryptoMetaData structure.
+ */
EncryptionAlgorithm encryption_algorithm;
+ /**
+ * Retrieval metadata of key used for signing the footer.
+ * Used only in encrypted files with plaintext footer.
+ */
std::string footer_signing_key_metadata;
_FileMetaData__isset __isset;
@@ -2853,8 +3751,8 @@ class FileMetaData : public virtual
::apache::thrift::TBase {
bool operator < (const FileMetaData & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
@@ -2868,16 +3766,29 @@ typedef struct _FileCryptoMetaData__isset {
bool key_metadata :1;
} _FileCryptoMetaData__isset;
+/**
+ * Crypto metadata for files with encrypted footer *
+ */
class FileCryptoMetaData : public virtual ::apache::thrift::TBase {
public:
FileCryptoMetaData(const FileCryptoMetaData&);
FileCryptoMetaData& operator=(const FileCryptoMetaData&);
- FileCryptoMetaData() : key_metadata() {
+ FileCryptoMetaData() noexcept
+ : key_metadata() {
}
virtual ~FileCryptoMetaData() noexcept;
+ /**
+ * Encryption algorithm. This field is only used for files
+ * with encrypted footer. Files with plaintext footer store algorithm id
+ * inside footer (FileMetaData structure).
+ */
EncryptionAlgorithm encryption_algorithm;
+ /**
+ * Retrieval metadata of key used for encryption of footer,
+ * and (possibly) columns *
+ */
std::string key_metadata;
_FileCryptoMetaData__isset __isset;
@@ -2902,8 +3813,8 @@ class FileCryptoMetaData : public virtual
::apache::thrift::TBase {
bool operator < (const FileCryptoMetaData & ) const;
- uint32_t read(::apache::thrift::protocol::TProtocol* iprot);
- uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const;
+ uint32_t read(::apache::thrift::protocol::TProtocol* iprot) override;
+ uint32_t write(::apache::thrift::protocol::TProtocol* oprot) const override;
virtual void printTo(std::ostream& out) const;
};
diff --git a/cpp/src/parquet/parquet.thrift b/cpp/src/parquet/parquet.thrift
index 8aa9848167..88e44c96cc 100644
--- a/cpp/src/parquet/parquet.thrift
+++ b/cpp/src/parquet/parquet.thrift
@@ -43,9 +43,10 @@ enum Type {
}
/**
- * Common types used by frameworks(e.g. hive, pig) using parquet. This helps
map
- * between types in those frameworks to the base types in parquet. This is
only
- * metadata and not needed to read or write the data.
+ * DEPRECATED: Common types used by frameworks(e.g. hive, pig) using parquet.
+ * ConvertedType is superseded by LogicalType. This enum should not be
extended.
+ *
+ * See LogicalTypes.md for conversion between ConvertedType and LogicalType.
*/
enum ConvertedType {
/** a BYTE_ARRAY actually contains UTF8 encoded chars */
@@ -318,15 +319,15 @@ struct BsonType {
* LogicalType annotations to replace ConvertedType.
*
* To maintain compatibility, implementations using LogicalType for a
- * SchemaElement must also set the corresponding ConvertedType from the
- * following table.
+ * SchemaElement must also set the corresponding ConvertedType (if any)
+ * from the following table.
*/
union LogicalType {
1: StringType STRING // use ConvertedType UTF8
2: MapType MAP // use ConvertedType MAP
3: ListType LIST // use ConvertedType LIST
4: EnumType ENUM // use ConvertedType ENUM
- 5: DecimalType DECIMAL // use ConvertedType DECIMAL
+ 5: DecimalType DECIMAL // use ConvertedType DECIMAL +
SchemaElement.{scale, precision}
6: DateType DATE // use ConvertedType DATE
// use ConvertedType TIME_MICROS for TIME(isAdjustedToUTC = *, unit = MICROS)
@@ -342,7 +343,7 @@ union LogicalType {
11: NullType UNKNOWN // no compatible ConvertedType
12: JsonType JSON // use ConvertedType JSON
13: BsonType BSON // use ConvertedType BSON
- 14: UUIDType UUID
+ 14: UUIDType UUID // no compatible ConvertedType
}
/**
@@ -355,7 +356,7 @@ struct SchemaElement {
/** Data type for this field. Not set if the current element is a non-leaf
node */
1: optional Type type;
- /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the vales.
+ /** If type is FIXED_LEN_BYTE_ARRAY, this is the byte length of the values.
* Otherwise, if specified, this is the maximum bit length to store any of
the values.
* (e.g. a low cardinality INT col could have this set to 3). Note that
this is
* in the schema, and therefore fixed for the entire file.
@@ -376,13 +377,19 @@ struct SchemaElement {
*/
5: optional i32 num_children;
- /** When the schema is the result of a conversion from another model
+ /**
+ * DEPRECATED: When the schema is the result of a conversion from another
model.
* Used to record the original type to help with cross conversion.
+ *
+ * This is superseded by logicalType.
*/
6: optional ConvertedType converted_type;
- /** Used when this column contains decimal data.
+ /**
+ * DEPRECATED: Used when this column contains decimal data.
* See the DECIMAL converted type for more details.
+ *
+ * This is superseded by using the DecimalType annotation in logicalType.
*/
7: optional i32 scale
8: optional i32 precision
@@ -529,6 +536,11 @@ struct IndexPageHeader {
// TODO
}
+/**
+ * The dictionary page must be placed at the first position of the column chunk
+ * if it is partly or completely dictionary encoded. At most one dictionary
page
+ * can be placed in a column chunk.
+ **/
struct DictionaryPageHeader {
/** Number of values in the dictionary **/
1: required i32 num_values;
@@ -629,32 +641,23 @@ struct PageHeader {
/** Compressed (and potentially encrypted) page size in bytes, not including
this header **/
3: required i32 compressed_page_size
- /** The 32bit CRC for the page, to be be calculated as follows:
- * - Using the standard CRC32 algorithm
- * - On the data only, i.e. this header should not be included. 'Data'
- * hereby refers to the concatenation of the repetition levels, the
- * definition levels and the column value, in this exact order.
- * - On the encoded versions of the repetition levels, definition levels and
- * column values
- * - On the compressed versions of the repetition levels, definition levels
- * and column values where possible;
- * - For v1 data pages, the repetition levels, definition levels and column
- * values are always compressed together. If a compression scheme is
- * specified, the CRC shall be calculated on the compressed version of
- * this concatenation. If no compression scheme is specified, the CRC
- * shall be calculated on the uncompressed version of this concatenation.
- * - For v2 data pages, the repetition levels and definition levels are
- * handled separately from the data and are never compressed (only
- * encoded). If a compression scheme is specified, the CRC shall be
- * calculated on the concatenation of the uncompressed repetition levels,
- * uncompressed definition levels and the compressed column values.
- * If no compression scheme is specified, the CRC shall be calculated on
- * the uncompressed concatenation.
- * - In encrypted columns, CRC is calculated after page encryption; the
- * encryption itself is performed after page compression (if compressed)
+ /** The 32-bit CRC checksum for the page, to be be calculated as follows:
+ *
+ * - The standard CRC32 algorithm is used (with polynomial 0x04C11DB7,
+ * the same as in e.g. GZip).
+ * - All page types can have a CRC (v1 and v2 data pages, dictionary pages,
+ * etc.).
+ * - The CRC is computed on the serialization binary representation of the
page
+ * (as written to disk), excluding the page header. For example, for v1
+ * data pages, the CRC is computed on the concatenation of repetition
levels,
+ * definition levels and column values (optionally compressed, optionally
+ * encrypted).
+ * - The CRC computation therefore takes place after any compression
+ * and encryption steps, if any.
+ *
* If enabled, this allows for disabling checksumming in HDFS if only a few
* pages need to be read.
- **/
+ */
4: optional i32 crc
// Headers for page specific data. One only will be set.
@@ -892,6 +895,13 @@ union ColumnOrder {
* - If the min is +0, the row group may contain -0 values as well.
* - If the max is -0, the row group may contain +0 values as well.
* - When looking for NaN values, min and max should be ignored.
+ *
+ * When writing statistics the following rules should be followed:
+ * - NaNs should not be written to min or max statistics fields.
+ * - If the computed max value is zero (whether negative or positive),
+ * `+0.0` should be written into the max statistics field.
+ * - If the computed min value is zero (whether negative or positive),
+ * `-0.0` should be written into the min statistics field.
*/
1: TypeDefinedOrder TYPE_ORDER;
}
@@ -936,19 +946,20 @@ struct ColumnIndex {
1: required list<bool> null_pages
/**
- * Two lists containing lower and upper bounds for the values of each page.
- * These may be the actual minimum and maximum values found on a page, but
- * can also be (more compact) values that do not exist on a page. For
- * example, instead of storing ""Blart Versenwald III", a writer may set
- * min_values[i]="B", max_values[i]="C". Such more compact values must still
- * be valid values within the column's logical type. Readers must make sure
- * that list entries are populated before using them by inspecting
null_pages.
+ * Two lists containing lower and upper bounds for the values of each page
+ * determined by the ColumnOrder of the column. These may be the actual
+ * minimum and maximum values found on a page, but can also be (more compact)
+ * values that do not exist on a page. For example, instead of storing
""Blart
+ * Versenwald III", a writer may set min_values[i]="B", max_values[i]="C".
+ * Such more compact values must still be valid values within the column's
+ * logical type. Readers must make sure that list entries are populated
before
+ * using them by inspecting null_pages.
*/
2: required list<binary> min_values
3: required list<binary> max_values
/**
- * Stores whether both min_values and max_values are orderd and if so, in
+ * Stores whether both min_values and max_values are ordered and if so, in
* which direction. This allows readers to perform binary searches in both
* lists. Readers cannot assume that max_values[i] <= min_values[i+1], even
* if the lists are ordered.
@@ -1019,17 +1030,20 @@ struct FileMetaData {
6: optional string created_by
/**
- * Sort order used for the min_value and max_value fields of each column in
- * this file. Sort orders are listed in the order matching the columns in the
- * schema. The indexes are not necessary the same though, because only leaf
- * nodes of the schema are represented in the list of sort orders.
+ * Sort order used for the min_value and max_value fields in the Statistics
+ * objects and the min_values and max_values fields in the ColumnIndex
+ * objects of each column in this file. Sort orders are listed in the order
+ * matching the columns in the schema. The indexes are not necessary the same
+ * though, because only leaf nodes of the schema are represented in the list
+ * of sort orders.
*
- * Without column_orders, the meaning of the min_value and max_value fields
is
- * undefined. To ensure well-defined behaviour, if min_value and max_value
are
- * written to a Parquet file, column_orders must be written as well.
+ * Without column_orders, the meaning of the min_value and max_value fields
+ * in the Statistics object and the ColumnIndex object is undefined. To
ensure
+ * well-defined behaviour, if these fields are written to a Parquet file,
+ * column_orders must be written as well.
*
- * The obsolete min and max fields are always sorted by signed comparison
- * regardless of column_orders.
+ * The obsolete min and max fields in the Statistics object are always sorted
+ * by signed comparison regardless of column_orders.
*/
7: optional list<ColumnOrder> column_orders;
