andygrove commented on code in PR #326:
URL: https://github.com/apache/datafusion-spark/pull/326#discussion_r4642980260
##########
crates/cli/src/main.rs:
##########
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible
functions
+//! registered and the Spark SQL dialect enabled.
+//!
+//! The shell itself (the REPL, running files and commands, output formats and
+//! object stores) comes from the `datafusion-cli` library.
+
+use std::env;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, absolute};
+use std::process::ExitCode;
+use std::sync::Arc;
+
+use clap::{ArgAction, Parser};
+use datafusion::common::config::{ConfigNonZeroUsize, Dialect};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::config::ConfigOptions;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::logical_expr::ExplainFormat;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_cli::catalog::DynamicObjectStoreCatalog;
+use datafusion_cli::exec;
+use datafusion_cli::object_storage::instrumented::{
+ InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,
+};
+use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location};
+use datafusion_cli::print_format::PrintFormat;
+use datafusion_cli::print_options::{MaxRows, PrintOptions};
+use datafusion_spark::SessionStateBuilderSpark;
+
+#[derive(Debug, Parser)]
+#[clap(version, about, long_about = None)]
+struct Args {
+ #[clap(
+ short = 'c',
+ long,
+ num_args = 0..,
+ help = "Execute the given command string(s), then exit",
+ value_parser(parse_command)
+ )]
+ command: Vec<String>,
+
+ #[clap(
+ short,
+ long,
+ num_args = 0..,
+ help = "Execute commands from file(s), then exit",
+ value_parser(parse_valid_file)
+ )]
+ file: Vec<String>,
+
+ #[clap(
+ short = 'r',
+ long,
+ num_args = 0..,
+ help = "Run the provided files on startup, before the REPL",
+ value_parser(parse_valid_file),
+ conflicts_with = "file"
+ )]
+ rc: Vec<String>,
+
+ #[clap(
+ short = 'p',
+ long,
+ help = "Path to your data, default to current directory",
+ value_parser(parse_valid_data_dir)
+ )]
+ data_path: Option<String>,
+
+ #[clap(
+ short = 'b',
+ long,
+ help = "The batch size of each query, or use DataFusion default",
+ value_parser(parse_batch_size)
+ )]
+ batch_size: Option<usize>,
+
+ #[clap(
+ long,
+ default_value_t = true,
+ action = ArgAction::Set,
+ help = "Evaluate with ANSI mode on, like Spark 4's default \
+ `spark.sql.ansi.enabled = true`. Pass `--ansi false` for
Spark's legacy behavior"
+ )]
+ ansi: bool,
+
+ #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)]
+ format: PrintFormat,
+
+ #[clap(
+ short,
+ long,
+ help = "Reduce printing other than the results and work quietly"
+ )]
+ quiet: bool,
+
+ #[clap(
+ long,
+ help = "The max number of rows to display for 'Table'
format\n[possible values: numbers(0/10/...), inf(no limit)]",
+ default_value = "40"
+ )]
+ maxrows: MaxRows,
+
+ #[clap(long, help = "Enables console syntax highlighting")]
+ color: bool,
+}
+
+impl Args {
+ /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from
stdin.
+ fn repl_mode(&self) -> bool {
+ self.command.is_empty() && self.file.is_empty()
+ }
+
+ /// Whether stdin carries the session's SQL, so it cannot also back a
+ /// `LOCATION '/dev/stdin'` table.
+ fn reads_sql_from_stdin(&self) -> bool {
+ self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f))
+ }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ if let Err(e) = main_inner().await {
+ println!("Error: {e}");
+ return ExitCode::FAILURE;
+ }
+ ExitCode::SUCCESS
+}
+
+async fn main_inner() -> Result<()> {
+ env_logger::init();
+ let mut args = Args::parse();
+
+ if !args.quiet {
+ println!(
+ "DataFusion Spark CLI v{} (ANSI mode {})",
+ env!("CARGO_PKG_VERSION"),
+ if args.ansi { "on" } else { "off" }
+ );
+ }
+
+ if let Some(path) = &args.data_path {
+ // `-f` and `-r` paths are relative to where the CLI was started, not
to
+ // the data directory.
+ args.file = absolute_paths(&args.file)?;
+ args.rc = absolute_paths(&args.rc)?;
+ env::set_current_dir(Path::new(path))?;
+ }
+
+ let instrumented_registry = Arc::new(
+ InstrumentedObjectStoreRegistry::new()
+ .with_profile_mode(InstrumentedObjectStoreMode::Disabled),
+ );
+ let runtime_env = RuntimeEnvBuilder::new()
+ .with_object_store_registry(Arc::clone(&instrumented_registry) as _)
+ .build_arc()?;
+ let state = SessionStateBuilder::new()
+ .with_config(session_config(&args)?)
+ .with_runtime_env(runtime_env)
+ .with_default_features()
+ .with_spark_features()
+ .build();
+
+ // Query files by their path or URL, such as `SELECT * FROM
'data.parquet'`, as
+ // `datafusion-cli` does.
+ let ctx = SessionContext::new_with_state(state).enable_url_table();
+ ctx.refresh_catalogs().await?;
+ ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(
+ ctx.state().catalog_list().clone(),
+ ctx.state_weak_ref(),
+ )));
+
+ let mut print_options = PrintOptions {
+ format: args.format,
+ quiet: args.quiet,
+ maxrows: args.maxrows,
+ color: args.color,
+ instrumented_registry,
+ };
+
+ if args.repl_mode() {
+ if !args.rc.is_empty() {
+ exec_files(&ctx, &args.rc, &print_options).await?;
+ }
+ return exec::exec_from_repl(&ctx, &mut print_options)
+ .await
+ .map_err(|e| DataFusionError::External(Box::new(e)));
+ }
+ if !args.file.is_empty() {
+ exec_files(&ctx, &args.file, &print_options).await?;
+ }
+ if !args.command.is_empty() {
+ exec::exec_from_commands(&ctx, args.command.clone(),
&print_options).await?;
+ }
+ Ok(())
+}
+
+/// Makes each path absolute, leaving the names of stdin as they are.
+fn absolute_paths(paths: &[String]) -> Result<Vec<String>> {
+ paths
+ .iter()
+ .map(|path| {
+ if is_stdin_location(path) {
+ return Ok(path.clone());
+ }
+ Ok(absolute(path)?.to_string_lossy().into_owned())
+ })
+ .collect()
+}
+
+/// Runs the SQL in each file in turn. Unlike `exec::exec_from_files`, which
panics,
+/// a file that cannot be opened is an error.
+async fn exec_files(
+ ctx: &SessionContext,
+ files: &[String],
+ print_options: &PrintOptions,
+) -> Result<()> {
+ for path in files {
Review Comment:
Good point, thanks. Fixed in 224e924. All the `-f` or `-r` files are now
opened into a `Vec<File>` before anything runs. CI has a new step where the
first file does a `COPY ... TO 'out.csv'` and the second is unreadable. It
checks for the open error and that `out.csv` was never written. That step fails
on the previous commit.
##########
crates/cli/src/main.rs:
##########
@@ -0,0 +1,285 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF)
under one or more contributor license agreements
+
+//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible
functions
+//! registered and the Spark SQL dialect enabled.
+//!
+//! The shell itself (the REPL, running files and commands, output formats and
+//! object stores) comes from the `datafusion-cli` library.
+
+use std::env;
+use std::fs::File;
+use std::io::BufReader;
+use std::path::{Path, absolute};
+use std::process::ExitCode;
+use std::sync::Arc;
+
+use clap::{ArgAction, Parser};
+use datafusion::common::config::{ConfigNonZeroUsize, Dialect};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::config::ConfigOptions;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::logical_expr::ExplainFormat;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_cli::catalog::DynamicObjectStoreCatalog;
+use datafusion_cli::exec;
+use datafusion_cli::object_storage::instrumented::{
+ InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,
+};
+use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location};
+use datafusion_cli::print_format::PrintFormat;
+use datafusion_cli::print_options::{MaxRows, PrintOptions};
+use datafusion_spark::SessionStateBuilderSpark;
+
+#[derive(Debug, Parser)]
+#[clap(version, about, long_about = None)]
+struct Args {
+ #[clap(
+ short = 'c',
+ long,
+ num_args = 0..,
+ help = "Execute the given command string(s), then exit",
+ value_parser(parse_command)
+ )]
+ command: Vec<String>,
+
+ #[clap(
+ short,
+ long,
+ num_args = 0..,
+ help = "Execute commands from file(s), then exit",
+ value_parser(parse_valid_file)
+ )]
+ file: Vec<String>,
+
+ #[clap(
+ short = 'r',
+ long,
+ num_args = 0..,
+ help = "Run the provided files on startup, before the REPL",
+ value_parser(parse_valid_file),
+ conflicts_with = "file"
+ )]
+ rc: Vec<String>,
+
+ #[clap(
+ short = 'p',
+ long,
+ help = "Path to your data, default to current directory",
+ value_parser(parse_valid_data_dir)
+ )]
+ data_path: Option<String>,
+
+ #[clap(
+ short = 'b',
+ long,
+ help = "The batch size of each query, or use DataFusion default",
+ value_parser(parse_batch_size)
+ )]
+ batch_size: Option<usize>,
+
+ #[clap(
+ long,
+ default_value_t = true,
+ action = ArgAction::Set,
+ help = "Evaluate with ANSI mode on, like Spark 4's default \
+ `spark.sql.ansi.enabled = true`. Pass `--ansi false` for
Spark's legacy behavior"
+ )]
+ ansi: bool,
+
+ #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)]
+ format: PrintFormat,
+
+ #[clap(
+ short,
+ long,
+ help = "Reduce printing other than the results and work quietly"
+ )]
+ quiet: bool,
+
+ #[clap(
+ long,
+ help = "The max number of rows to display for 'Table'
format\n[possible values: numbers(0/10/...), inf(no limit)]",
+ default_value = "40"
+ )]
+ maxrows: MaxRows,
+
+ #[clap(long, help = "Enables console syntax highlighting")]
+ color: bool,
+}
+
+impl Args {
+ /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from
stdin.
+ fn repl_mode(&self) -> bool {
+ self.command.is_empty() && self.file.is_empty()
+ }
+
+ /// Whether stdin carries the session's SQL, so it cannot also back a
+ /// `LOCATION '/dev/stdin'` table.
+ fn reads_sql_from_stdin(&self) -> bool {
+ self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f))
+ }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+ if let Err(e) = main_inner().await {
+ println!("Error: {e}");
+ return ExitCode::FAILURE;
+ }
+ ExitCode::SUCCESS
+}
+
+async fn main_inner() -> Result<()> {
+ env_logger::init();
+ let mut args = Args::parse();
+
+ if !args.quiet {
+ println!(
+ "DataFusion Spark CLI v{} (ANSI mode {})",
+ env!("CARGO_PKG_VERSION"),
+ if args.ansi { "on" } else { "off" }
+ );
+ }
+
+ if let Some(path) = &args.data_path {
+ // `-f` and `-r` paths are relative to where the CLI was started, not
to
+ // the data directory.
+ args.file = absolute_paths(&args.file)?;
+ args.rc = absolute_paths(&args.rc)?;
+ env::set_current_dir(Path::new(path))?;
+ }
+
+ let instrumented_registry = Arc::new(
+ InstrumentedObjectStoreRegistry::new()
+ .with_profile_mode(InstrumentedObjectStoreMode::Disabled),
+ );
+ let runtime_env = RuntimeEnvBuilder::new()
+ .with_object_store_registry(Arc::clone(&instrumented_registry) as _)
+ .build_arc()?;
+ let state = SessionStateBuilder::new()
+ .with_config(session_config(&args)?)
+ .with_runtime_env(runtime_env)
+ .with_default_features()
+ .with_spark_features()
+ .build();
+
+ // Query files by their path or URL, such as `SELECT * FROM
'data.parquet'`, as
+ // `datafusion-cli` does.
+ let ctx = SessionContext::new_with_state(state).enable_url_table();
+ ctx.refresh_catalogs().await?;
+ ctx.register_catalog_list(Arc::new(DynamicObjectStoreCatalog::new(
+ ctx.state().catalog_list().clone(),
+ ctx.state_weak_ref(),
+ )));
+
+ let mut print_options = PrintOptions {
+ format: args.format,
+ quiet: args.quiet,
+ maxrows: args.maxrows,
+ color: args.color,
+ instrumented_registry,
+ };
+
+ if args.repl_mode() {
+ if !args.rc.is_empty() {
+ exec_files(&ctx, &args.rc, &print_options).await?;
+ }
+ return exec::exec_from_repl(&ctx, &mut print_options)
+ .await
+ .map_err(|e| DataFusionError::External(Box::new(e)));
+ }
+ if !args.file.is_empty() {
+ exec_files(&ctx, &args.file, &print_options).await?;
+ }
+ if !args.command.is_empty() {
+ exec::exec_from_commands(&ctx, args.command.clone(),
&print_options).await?;
+ }
+ Ok(())
+}
+
+/// Makes each path absolute, leaving the names of stdin as they are.
+fn absolute_paths(paths: &[String]) -> Result<Vec<String>> {
+ paths
+ .iter()
+ .map(|path| {
+ if is_stdin_location(path) {
+ return Ok(path.clone());
+ }
+ Ok(absolute(path)?.to_string_lossy().into_owned())
Review Comment:
Fixed in 224e924 the way you suggested. The files are opened before changing
to the data directory, so `absolute_paths` and its `to_string_lossy()` are gone
and nothing converts the path.
##########
.github/workflows/cli.yml:
##########
@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under
one or more contributor license agreements
+
+name: CLI
+
+on:
+ push:
+ branches: [main]
+ pull_request:
+
+permissions:
+ contents: read
+
+jobs:
+ smoke-test:
+ name: datafusion-spark-cli smoke test
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v4
+ - run: rustup toolchain install stable --profile minimal
--no-self-update && rustup default stable
+ - uses: Swatinem/rust-cache@v2
+ - run: cargo build -p datafusion-spark-cli
+ # A Spark function, and ANSI mode on by default and off with `--ansi
false`.
+ - name: Run queries
+ run: |
+ set -euo pipefail
+ cli=target/debug/datafusion-spark-cli
+ $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \
+ | grep -x
'529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b'
+ if out=$($cli -q -c "SELECT abs(cast(-2147483648 AS int))" 2>&1);
then
+ echo "expected an ARITHMETIC_OVERFLOW error with ANSI mode on" &&
exit 1
+ fi
+ grep -F '[ARITHMETIC_OVERFLOW]' <<< "$out"
+ $cli -q --ansi false --format csv -c "SELECT abs(cast(-2147483648 AS
int)) AS a" \
+ | grep -x -- '-2147483648'
+ # SQL read from stdin by the REPL. `LONG` is a type in the Spark dialect
only.
+ - name: Run the REPL from stdin
+ run: |
+ set -euo pipefail
+ echo "SELECT CAST(1 AS LONG) AS l;" \
+ | target/debug/datafusion-spark-cli -q --format csv | grep -x '1'
+ # `-f` and `-r` paths are relative to the working directory, not to `-p`.
+ - name: Run files with a data path
+ run: |
+ set -euo pipefail
+ cli=$PWD/target/debug/datafusion-spark-cli
+ dir=$(mktemp -d) && cd "$dir" && mkdir data
+ echo "SELECT 'cwd' AS s;" > q.sql
+ echo "SELECT 'data' AS s;" > data/q.sql
+ $cli -q --format csv -p data -f q.sql | grep -x 'cwd'
+ echo "" | $cli -q --format csv -p data -r q.sql | grep -x 'cwd'
+ echo "SELECT 'stdin' AS s;" | $cli -q --format csv -p data -f
/dev/stdin | grep -x 'stdin'
Review Comment:
Added in 224e924. The launch directory and `data/` each have a `t.csv` with
different contents, and `data/` also has a decoy `csv.sql`. The script is run
with `-p data`. CI checks that the script runs from the launch directory and
reads `data/t.csv`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]