scovich commented on code in PR #7670:
URL: https://github.com/apache/arrow-rs/pull/7670#discussion_r2165054044
##########
parquet-variant/src/variant.rs:
##########
@@ -1009,6 +1009,14 @@ impl From<VariantDecimal4> for Variant<'_, '_> {
Variant::Decimal4(value)
}
}
+impl From<bool> for Variant<'_, '_> {
+ fn from(value: bool) -> Self {
+ match value {
+ true => Variant::BooleanTrue,
+ false => Variant::BooleanFalse,
+ }
+ }
+}
Review Comment:
Why move it here, out of curiosity? (between decimal4 and decimal8)? Was
there some merge conflict?
(the original location wasn't amazing either... but if we're going to move
it why not move to near the top, below the `impl From<()>`?)
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
Review Comment:
This is begging for a generic method... three syntactically ~identical
copies of some rather complex code. I don't know a good way to do it, tho
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
Review Comment:
simpler to check for `remainder == 0`, before formatting?
```rust
if remainder == 0 {
write!(json_buffer, "{quotient}")?;
} else {
let remainder = format!("{remainder:0width$}", width = *scale as usize)
let remainder = remainder.trim_end_matches('0');
write!(json_buffer, "{quotient}.{remainder}")?;
}
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
+}
+
+/// Convert Variant to serde_json::Value
+///
+/// This function converts a Variant to a [`serde_json::Value`], which is
useful
+/// when you need to work with the JSON data programmatically or integrate with
+/// other serde-based JSON processing.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(Value)` containing the JSON value
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::Number(42.into()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("hello");
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::String("hello".to_string()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
+ match variant {
+ Variant::Null => Ok(Value::Null),
+ Variant::BooleanTrue => Ok(Value::Bool(true)),
+ Variant::BooleanFalse => Ok(Value::Bool(false)),
+ Variant::Int8(i) => Ok(Value::Number((*i).into())),
+ Variant::Int16(i) => Ok(Value::Number((*i).into())),
+ Variant::Int32(i) => Ok(Value::Number((*i).into())),
+ Variant::Int64(i) => Ok(Value::Number((*i).into())),
+ Variant::Float(f) => serde_json::Number::from_f64(*f as f64)
+ .map(Value::Number)
+ .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid float
value".to_string())),
+ Variant::Double(f) => serde_json::Number::from_f64(*f)
+ .map(Value::Number)
+ .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid double
value".to_string())),
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer).into()))
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
+ decimal_str
+ .parse::<serde_json::Number>()
+ .map(Value::Number)
+ .map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("Invalid
decimal string: {}", e))
+ })
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer).into()))
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
+ decimal_str
+ .parse::<serde_json::Number>()
+ .map(Value::Number)
+ .map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("Invalid
decimal string: {}", e))
+ })
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer as i64).into())) // Convert to i64
for JSON compatibility
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
+ decimal_str
+ .parse::<serde_json::Number>()
+ .map(Value::Number)
+ .map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("Invalid
decimal string: {}", e))
+ })
+ }
+ }
+ Variant::Date(date) => Ok(Value::String(format_date_string(date))),
+ Variant::TimestampMicros(ts) => Ok(Value::String(ts.to_rfc3339())),
+ Variant::TimestampNtzMicros(ts) =>
Ok(Value::String(format_timestamp_ntz_string(ts))),
+ Variant::Binary(bytes) =>
Ok(Value::String(format_binary_base64(bytes))),
+ Variant::String(s) => Ok(Value::String(s.to_string())),
+ Variant::ShortString(s) => Ok(Value::String(s.to_string())),
+ Variant::Object(obj) => {
+ let mut map = serde_json::Map::new();
+
+ for (key, value) in obj.iter() {
+ let json_value = variant_to_json_value(&value)?;
+ map.insert(key.to_string(), json_value);
+ }
+
+ Ok(Value::Object(map))
+ }
+ Variant::List(arr) => {
+ let mut vec = Vec::new();
+ let len = arr.len();
+
+ for i in 0..len {
+ let element = arr.get(i)?;
+ let json_value = variant_to_json_value(&element)?;
+ vec.push(json_value);
+ }
+
Review Comment:
similar to above:
```suggestion
let vec = arr
.iter()
.map(|element| variant_to_json_value(&element))
.collect::<Result<_, _>>();
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
+}
+
+/// Convert Variant to serde_json::Value
+///
+/// This function converts a Variant to a [`serde_json::Value`], which is
useful
+/// when you need to work with the JSON data programmatically or integrate with
+/// other serde-based JSON processing.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(Value)` containing the JSON value
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::Number(42.into()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("hello");
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::String("hello".to_string()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
+ match variant {
+ Variant::Null => Ok(Value::Null),
+ Variant::BooleanTrue => Ok(Value::Bool(true)),
+ Variant::BooleanFalse => Ok(Value::Bool(false)),
+ Variant::Int8(i) => Ok(Value::Number((*i).into())),
+ Variant::Int16(i) => Ok(Value::Number((*i).into())),
+ Variant::Int32(i) => Ok(Value::Number((*i).into())),
+ Variant::Int64(i) => Ok(Value::Number((*i).into())),
+ Variant::Float(f) => serde_json::Number::from_f64(*f as f64)
+ .map(Value::Number)
+ .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid float
value".to_string())),
+ Variant::Double(f) => serde_json::Number::from_f64(*f)
+ .map(Value::Number)
+ .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid double
value".to_string())),
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer).into()))
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
Review Comment:
This is an interesting conundrum... I don't think parsing actually preserves
precision at all, other than the difference between `f64` and `i64`? Also,
`serde_json::Value` provides an extensive set of `impl From`:
```rust
let integer = if *scale == 0 {
*integer
} else {
let divisor = ...;
if integer % divisor != 0 {
// fall back to floating point
return Ok(Value::From(integer as f64 / divisor as f64))
}
integer / divisor
}
Ok(Value::from(integer));
```
The above works for i32 and i64, but i128 would need somewhat different
handling of the final `integer`:
```rust
// Prefer to emit as i64, but fall back to u64 or even f64 if necessary
i64::try_from(integer)
.map(Value::from)
.or_else(|| u64::try_from(integer))
.map(Value::from)
.unwrap_or_else(|| Value::from(integer as f64))
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
Review Comment:
`write!` returns `Result<()>`, so should be able to simplify a bunch of
these:
```suggestion
Variant::Null => write!(json_buffer, "null")?,
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
Review Comment:
This is clever enough to merit a code comment? Given a remainder like `100`:
* The `{remainder:0width$}` format ensures it correctly renders with leading
zeros (e.g. `.00000100`)
* The `trime_end_matches` then strips away any trailing zeros (e.g.
`.000001`)
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
Review Comment:
String also implements Write:
```suggestion
let mut string = String::new();
variant_to_json(&mut string, variant)?;
Ok(string)
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
Review Comment:
```suggestion
let first = true;
for element in arr.iter() {
if !first {
write!(buffer, ",")?;
}
first = false;
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
+}
+
+/// Convert Variant to serde_json::Value
+///
+/// This function converts a Variant to a [`serde_json::Value`], which is
useful
+/// when you need to work with the JSON data programmatically or integrate with
+/// other serde-based JSON processing.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(Value)` containing the JSON value
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::Number(42.into()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("hello");
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::String("hello".to_string()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
+ match variant {
+ Variant::Null => Ok(Value::Null),
+ Variant::BooleanTrue => Ok(Value::Bool(true)),
+ Variant::BooleanFalse => Ok(Value::Bool(false)),
+ Variant::Int8(i) => Ok(Value::Number((*i).into())),
+ Variant::Int16(i) => Ok(Value::Number((*i).into())),
+ Variant::Int32(i) => Ok(Value::Number((*i).into())),
+ Variant::Int64(i) => Ok(Value::Number((*i).into())),
+ Variant::Float(f) => serde_json::Number::from_f64(*f as f64)
Review Comment:
```suggestion
Variant::Float(f) => serde_json::Number::from_f64((*f).into())
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
Review Comment:
This should work:
```rust
fn write_decimal(
json_buffer: &mut impl Write,
integer: impl Into<i128>,
scale: u8,
) -> Result<(), ArrowError> {
let integer = integer.into();
// ... the current Decimal16 logic ...
}
```
and then all three call sites look the same:
```rust
write_decimal(json_buffer, *integer, *scale)
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
+}
+
+/// Convert Variant to serde_json::Value
+///
+/// This function converts a Variant to a [`serde_json::Value`], which is
useful
+/// when you need to work with the JSON data programmatically or integrate with
+/// other serde-based JSON processing.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(Value)` containing the JSON value
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::Number(42.into()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("hello");
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::String("hello".to_string()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
+ match variant {
+ Variant::Null => Ok(Value::Null),
+ Variant::BooleanTrue => Ok(Value::Bool(true)),
+ Variant::BooleanFalse => Ok(Value::Bool(false)),
+ Variant::Int8(i) => Ok(Value::Number((*i).into())),
+ Variant::Int16(i) => Ok(Value::Number((*i).into())),
+ Variant::Int32(i) => Ok(Value::Number((*i).into())),
+ Variant::Int64(i) => Ok(Value::Number((*i).into())),
+ Variant::Float(f) => serde_json::Number::from_f64(*f as f64)
Review Comment:
Should NaN/Inf be rendered as `Variant::Null`? Seems a bit harsh to just
fail the conversion of a deeply nested variant because a stray divide by zero
slipped in somewhere?
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
+}
+
+/// Convert Variant to serde_json::Value
+///
+/// This function converts a Variant to a [`serde_json::Value`], which is
useful
+/// when you need to work with the JSON data programmatically or integrate with
+/// other serde-based JSON processing.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(Value)` containing the JSON value
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::Number(42.into()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("hello");
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::String("hello".to_string()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
+ match variant {
+ Variant::Null => Ok(Value::Null),
+ Variant::BooleanTrue => Ok(Value::Bool(true)),
+ Variant::BooleanFalse => Ok(Value::Bool(false)),
+ Variant::Int8(i) => Ok(Value::Number((*i).into())),
+ Variant::Int16(i) => Ok(Value::Number((*i).into())),
+ Variant::Int32(i) => Ok(Value::Number((*i).into())),
+ Variant::Int64(i) => Ok(Value::Number((*i).into())),
+ Variant::Float(f) => serde_json::Number::from_f64(*f as f64)
+ .map(Value::Number)
+ .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid float
value".to_string())),
+ Variant::Double(f) => serde_json::Number::from_f64(*f)
+ .map(Value::Number)
+ .ok_or_else(|| ArrowError::InvalidArgumentError("Invalid double
value".to_string())),
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer).into()))
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
+ decimal_str
+ .parse::<serde_json::Number>()
+ .map(Value::Number)
+ .map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("Invalid
decimal string: {}", e))
+ })
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer).into()))
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
+ decimal_str
+ .parse::<serde_json::Number>()
+ .map(Value::Number)
+ .map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("Invalid
decimal string: {}", e))
+ })
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Use integer arithmetic to avoid f64 precision loss
+ if *scale == 0 {
+ Ok(Value::Number((*integer as i64).into())) // Convert to i64
for JSON compatibility
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+
+ let decimal_str = if trimmed_remainder.is_empty() {
+ quotient.to_string()
+ } else {
+ format!("{}.{}", quotient, trimmed_remainder)
+ };
+
+ // Parse as serde_json::Number to preserve precision
+ decimal_str
+ .parse::<serde_json::Number>()
+ .map(Value::Number)
+ .map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("Invalid
decimal string: {}", e))
+ })
+ }
+ }
+ Variant::Date(date) => Ok(Value::String(format_date_string(date))),
+ Variant::TimestampMicros(ts) => Ok(Value::String(ts.to_rfc3339())),
+ Variant::TimestampNtzMicros(ts) =>
Ok(Value::String(format_timestamp_ntz_string(ts))),
+ Variant::Binary(bytes) =>
Ok(Value::String(format_binary_base64(bytes))),
+ Variant::String(s) => Ok(Value::String(s.to_string())),
+ Variant::ShortString(s) => Ok(Value::String(s.to_string())),
+ Variant::Object(obj) => {
+ let mut map = serde_json::Map::new();
+
+ for (key, value) in obj.iter() {
+ let json_value = variant_to_json_value(&value)?;
+ map.insert(key.to_string(), json_value);
+ }
+
Review Comment:
`serde_json::Map` implements
[FromIterator](https://docs.rs/serde_json/latest/serde_json/struct.Map.html#impl-FromIterator%3C(String,+Value)%3E-for-Map%3CString,+Value%3E):
```suggestion
let map = obj
.iter
.map(|(k, v)| Ok((k.to_string(),
variant_to_json_value(&v)?)))
.collect::<Result<_, _>>()?;
```
##########
parquet-variant/src/to_json.rs:
##########
@@ -0,0 +1,1272 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+//! Module for converting Variant data to JSON format
+
+use arrow_schema::ArrowError;
+use base64::{engine::general_purpose, Engine as _};
+use serde_json::Value;
+use std::io::Write;
+
+use crate::variant::{Variant, VariantList, VariantObject};
+use crate::{VariantDecimal16, VariantDecimal4, VariantDecimal8};
+
+// Format string constants to avoid duplication and reduce errors
+const DATE_FORMAT: &str = "%Y-%m-%d";
+const TIMESTAMP_NTZ_FORMAT: &str = "%Y-%m-%dT%H:%M:%S%.6f";
+
+// Helper functions for consistent formatting
+fn format_date_string(date: &chrono::NaiveDate) -> String {
+ date.format(DATE_FORMAT).to_string()
+}
+
+fn format_timestamp_ntz_string(ts: &chrono::NaiveDateTime) -> String {
+ ts.format(TIMESTAMP_NTZ_FORMAT).to_string()
+}
+
+fn format_binary_base64(bytes: &[u8]) -> String {
+ general_purpose::STANDARD.encode(bytes)
+}
+
+/// Converts a Variant to JSON and writes it to the provided `Write`
+///
+/// This function writes JSON directly to any type that implements [`Write`],
+/// making it efficient for streaming or when you want to control the output
destination.
+///
+/// # Arguments
+///
+/// * `json_buffer` - Writer to output JSON to
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(())` if successful
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let mut buffer = Vec::new();
+/// variant_to_json(&mut buffer, &variant)?;
+/// assert_eq!(String::from_utf8(buffer).unwrap(), "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json(json_buffer: &mut impl Write, variant: &Variant) ->
Result<(), ArrowError> {
+ match variant {
+ Variant::Null => {
+ write!(json_buffer, "null")?;
+ }
+ Variant::BooleanTrue => {
+ write!(json_buffer, "true")?;
+ }
+ Variant::BooleanFalse => {
+ write!(json_buffer, "false")?;
+ }
+ Variant::Int8(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int16(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int32(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Int64(i) => {
+ write!(json_buffer, "{}", i)?;
+ }
+ Variant::Float(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Double(f) => {
+ write!(json_buffer, "{}", f)?;
+ }
+ Variant::Decimal4(VariantDecimal4 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i32.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal8(VariantDecimal8 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i64.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Decimal16(VariantDecimal16 { integer, scale }) => {
+ // Convert decimal to string representation using integer
arithmetic
+ if *scale == 0 {
+ write!(json_buffer, "{}", integer)?;
+ } else {
+ let divisor = 10_i128.pow(*scale as u32);
+ let quotient = integer / divisor;
+ let remainder = (integer % divisor).abs();
+ let formatted_remainder = format!("{:0width$}", remainder,
width = *scale as usize);
+ let trimmed_remainder =
formatted_remainder.trim_end_matches('0');
+ if trimmed_remainder.is_empty() {
+ write!(json_buffer, "{}", quotient)?;
+ } else {
+ write!(json_buffer, "{}.{}", quotient, trimmed_remainder)?;
+ }
+ }
+ }
+ Variant::Date(date) => {
+ write!(json_buffer, "\"{}\"", format_date_string(date))?;
+ }
+ Variant::TimestampMicros(ts) => {
+ write!(json_buffer, "\"{}\"", ts.to_rfc3339())?;
+ }
+ Variant::TimestampNtzMicros(ts) => {
+ write!(json_buffer, "\"{}\"", format_timestamp_ntz_string(ts))?;
+ }
+ Variant::Binary(bytes) => {
+ // Encode binary as base64 string
+ let base64_str = format_binary_base64(bytes);
+ let json_str = serde_json::to_string(&base64_str).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::String(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::ShortString(s) => {
+ // Use serde_json to properly escape the string
+ let json_str = serde_json::to_string(s.as_str()).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON encoding error:
{}", e))
+ })?;
+ write!(json_buffer, "{}", json_str)?;
+ }
+ Variant::Object(obj) => {
+ convert_object_to_json(json_buffer, obj)?;
+ }
+ Variant::List(arr) => {
+ convert_array_to_json(json_buffer, arr)?;
+ }
+ }
+ Ok(())
+}
+
+/// Convert object fields to JSON
+fn convert_object_to_json(buffer: &mut impl Write, obj: &VariantObject) ->
Result<(), ArrowError> {
+ write!(buffer, "{{")?;
+
+ // Get all fields from the object
+ let mut first = true;
+
+ for (key, value) in obj.iter() {
+ if !first {
+ write!(buffer, ",")?;
+ }
+ first = false;
+
+ // Write the key (properly escaped)
+ let json_key = serde_json::to_string(key).map_err(|e| {
+ ArrowError::InvalidArgumentError(format!("JSON key encoding error:
{}", e))
+ })?;
+ write!(buffer, "{}:", json_key)?;
+
+ // Recursively convert the value
+ variant_to_json(buffer, &value)?;
+ }
+
+ write!(buffer, "}}")?;
+ Ok(())
+}
+
+/// Convert array elements to JSON
+fn convert_array_to_json(buffer: &mut impl Write, arr: &VariantList) ->
Result<(), ArrowError> {
+ write!(buffer, "[")?;
+
+ let len = arr.len();
+ for i in 0..len {
+ if i > 0 {
+ write!(buffer, ",")?;
+ }
+
+ let element = arr.get(i)?;
+ variant_to_json(buffer, &element)?;
+ }
+
+ write!(buffer, "]")?;
+ Ok(())
+}
+
+/// Convert Variant to JSON string
+///
+/// This is a convenience function that converts a Variant to a JSON string.
+/// This is the same as calling variant_to_json with a Vec
+/// It's the simplest way to get a JSON representation when you just need a
String result.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(String)` containing the JSON representation
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "42");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("Hello, World!");
+/// let json = variant_to_json_string(&variant)?;
+/// assert_eq!(json, "\"Hello, World!\"");
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// # Example: Create a [`Variant::Object`] and convert to JSON
+///
+/// This example shows how to create an object with two fields and convert it
to JSON:
+/// ```json
+/// {
+/// "first_name": "Jiaying",
+/// "last_name": "Li"
+/// }
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, VariantBuilder, variant_to_json_string};
+/// # use arrow_schema::ArrowError;
+/// let mut builder = VariantBuilder::new();
+/// // Create an object builder that will write fields to the object
+/// let mut object_builder = builder.new_object();
+/// object_builder.append_value("first_name", "Jiaying");
+/// object_builder.append_value("last_name", "Li");
+/// object_builder.finish();
+/// // Finish the builder to get the metadata and value
+/// let (metadata, value) = builder.finish();
+/// // Create the Variant and convert to JSON
+/// let variant = Variant::try_new(&metadata, &value)?;
+/// let json = variant_to_json_string(&variant)?;
+/// assert!(json.contains("\"first_name\":\"Jiaying\""));
+/// assert!(json.contains("\"last_name\":\"Li\""));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_string(variant: &Variant) -> Result<String, ArrowError>
{
+ let mut buffer = Vec::new();
+ variant_to_json(&mut buffer, variant)?;
+ String::from_utf8(buffer)
+ .map_err(|e| ArrowError::InvalidArgumentError(format!("UTF-8
conversion error: {}", e)))
+}
+
+/// Convert Variant to serde_json::Value
+///
+/// This function converts a Variant to a [`serde_json::Value`], which is
useful
+/// when you need to work with the JSON data programmatically or integrate with
+/// other serde-based JSON processing.
+///
+/// # Arguments
+///
+/// * `variant` - The Variant value to convert
+///
+/// # Returns
+///
+/// * `Ok(Value)` containing the JSON value
+/// * `Err` with error details if conversion fails
+///
+/// # Examples
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::Int32(42);
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::Number(42.into()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+///
+/// ```rust
+/// # use parquet_variant::{Variant, variant_to_json_value};
+/// # use serde_json::Value;
+/// # use arrow_schema::ArrowError;
+/// let variant = Variant::String("hello");
+/// let json_value = variant_to_json_value(&variant)?;
+/// assert_eq!(json_value, Value::String("hello".to_string()));
+/// # Ok::<(), ArrowError>(())
+/// ```
+pub fn variant_to_json_value(variant: &Variant) -> Result<Value, ArrowError> {
+ match variant {
+ Variant::Null => Ok(Value::Null),
+ Variant::BooleanTrue => Ok(Value::Bool(true)),
+ Variant::BooleanFalse => Ok(Value::Bool(false)),
+ Variant::Int8(i) => Ok(Value::Number((*i).into())),
+ Variant::Int16(i) => Ok(Value::Number((*i).into())),
+ Variant::Int32(i) => Ok(Value::Number((*i).into())),
+ Variant::Int64(i) => Ok(Value::Number((*i).into())),
+ Variant::Float(f) => serde_json::Number::from_f64(*f as f64)
Review Comment:
Note: [`impl From<f64> for
serde_json::Value`](https://docs.rs/serde_json/latest/serde_json/enum.Value.html#impl-From%3Cf64%3E-for-Value)
converts them to `Value::Null`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
