alamb commented on code in PR #51:
URL: https://github.com/apache/datafusion-spark/pull/51#discussion_r4544793786
##########
crates/sqllogictest/tests/sqllogictests.rs:
##########
@@ -0,0 +1,106 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! Runs the `.slt` files under `test_files` against a DataFusion session with
the Spark
+//! functions registered and the Spark SQL dialect enabled.
+//!
+//! Usage: `cargo test -p datafusion-spark-sqllogictest -- [FILTER]`, where
`FILTER` is an
+//! optional substring of the test file path.
+
+use std::path::{Path, PathBuf};
+use std::process::ExitCode;
+
+use datafusion::common::config::Dialect;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_spark::SessionStateBuilderSpark;
+use datafusion_sqllogictest::{DataFusion, df_value_validator,
value_normalizer};
+use indicatif::ProgressBar;
+use sqllogictest::strict_column_validator;
+
+fn collect_slt_files(dir: &Path, files: &mut Vec<PathBuf>) ->
std::io::Result<()> {
+ for entry in std::fs::read_dir(dir)? {
+ let path = entry?.path();
+ if path.is_dir() {
+ collect_slt_files(&path, files)?;
+ } else if path.extension().is_some_and(|ext| ext == "slt") {
+ files.push(path);
+ }
+ }
+ Ok(())
+}
+
+fn new_context() -> SessionContext {
+ let mut config = SessionConfig::new().with_target_partitions(4);
+ config.options_mut().sql_parser.dialect = Dialect::Spark;
+ let state = SessionStateBuilder::new()
+ .with_config(config)
+ .with_default_features()
+ .with_spark_features()
+ .build();
+ SessionContext::new_with_state(state)
+}
+
+async fn run_file(path: PathBuf, relative_path: PathBuf) -> Result<(), String>
{
+ // The DataFusion engine formats floats the way Spark does for paths under
`spark/`
+ let engine_path = Path::new("spark").join(&relative_path);
+ let mut runner = sqllogictest::Runner::new(|| async {
+ Ok(DataFusion::new(
+ new_context(),
+ engine_path.clone(),
+ ProgressBar::hidden(),
+ ))
+ });
+ runner.with_column_validator(strict_column_validator);
+ runner.with_normalizer(value_normalizer);
+ runner.with_validator(df_value_validator);
+ runner
+ .run_file_async(&path)
+ .await
+ .map_err(|e| e.to_string())
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ // Ignore flags such as `--nocapture` that `cargo test` may pass through.
+ let filters: Vec<String> = std::env::args()
+ .skip(1)
+ .filter(|arg| !arg.starts_with('-'))
+ .collect();
+
+ let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("test_files");
+ let mut files = vec![];
+ collect_slt_files(&root, &mut files).expect("failed to list test files");
+ files.sort();
+
+ let mut handles = vec![];
Review Comment:
this is a pretty clean runner
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
