kosiew commented on code in PR #326:
URL: https://github.com/apache/datafusion-spark/pull/326#discussion_r4626643698


##########
crates/spark/README.md:
##########
@@ -66,17 +66,18 @@ use datafusion_spark::expr_fn::sha2;
 let expr = sha2(col("my_data"), lit(256));
 ```
 
-## Using the DataFusion CLI
+## Trying the functions in a SQL shell
 
-Pass `--spark` to `datafusion-cli` to enable the Spark SQL dialect and register
-the Spark-compatible scalar, aggregate, and window functions for the session:
+`datafusion-spark-cli` is the DataFusion CLI with the Spark SQL dialect enabled
+and the Spark-compatible scalar, aggregate, and window functions registered. It
+lives in [`crates/cli`](../cli) in this repository and is not published yet:
 
 ```shell
-datafusion-cli --spark
+cargo run -p datafusion-spark-cli
 ```
 
-The option is disabled by default because Spark-compatible functions can
-override DataFusion functions with the same name.
+The Spark functions override DataFusion functions with the same name. ANSI mode
+is on by default, as in Spark 4. Pass `--ansi false` for Spark's legacy 
behavior.

Review Comment:
   Please document `cargo run -p datafusion-spark-cli -- --ansi false` here and 
in the SQLLogicTest README. The `--` separator is required to pass `--ansi` to 
the CLI.



##########
.github/workflows/cli.yml:
##########
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+name: CLI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  smoke-test:
+    name: datafusion-spark-cli smoke test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install stable --profile minimal 
--no-self-update && rustup default stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build -p datafusion-spark-cli
+      # A Spark function, and ANSI mode on by default and off with `--ansi 
false`.
+      - name: Run queries
+        run: |
+          set -euo pipefail
+          cli=target/debug/datafusion-spark-cli
+          $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \

Review Comment:
   Could we add a stdin REPL smoke test using `CAST(1 AS LONG)` and assert the 
result row? Current queries also pass with the generic dialect, so they do not 
verify Spark parsing.



##########
.github/workflows/cli.yml:
##########
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under 
one or more contributor license agreements
+
+name: CLI
+
+on:
+  push:
+    branches: [main]
+  pull_request:
+
+permissions:
+  contents: read
+
+jobs:
+  smoke-test:
+    name: datafusion-spark-cli smoke test
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - run: rustup toolchain install stable --profile minimal 
--no-self-update && rustup default stable
+      - uses: Swatinem/rust-cache@v2
+      - run: cargo build -p datafusion-spark-cli
+      # A Spark function, and ANSI mode on by default and off with `--ansi 
false`.
+      - name: Run queries
+        run: |
+          set -euo pipefail
+          cli=target/debug/datafusion-spark-cli
+          $cli -q --format csv -c "SELECT sha2('Spark', 256) AS h" \
+            | grep -x 
'529bc3b07127ecb7e53a4dcf1991d9152c24537d919178022b2c42657f79a26b'
+          if $cli -q -c "SELECT abs(cast(-2147483648 AS int))"; then

Review Comment:
   Please also assert `[ARITHMETIC_OVERFLOW]` in the captured output. A startup 
failure or panic could otherwise satisfy the nonzero-exit check.



##########
crates/cli/src/main.rs:
##########
@@ -0,0 +1,249 @@
+// SPDX-License-Identifier: Apache-2.0
+// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for 
additional information regarding copyright ownership
+// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) 
under one or more contributor license agreements
+
+//! `datafusion-spark-cli`: the DataFusion CLI with the Spark-compatible 
functions
+//! registered and the Spark SQL dialect enabled.
+//!
+//! The shell itself (the REPL, running files and commands, output formats and
+//! object stores) comes from the `datafusion-cli` library.
+
+use std::env;
+use std::path::Path;
+use std::process::ExitCode;
+use std::sync::Arc;
+
+use clap::{ArgAction, Parser};
+use datafusion::common::config::{ConfigNonZeroUsize, Dialect};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::config::ConfigOptions;
+use datafusion::execution::SessionStateBuilder;
+use datafusion::execution::runtime_env::RuntimeEnvBuilder;
+use datafusion::logical_expr::ExplainFormat;
+use datafusion::prelude::{SessionConfig, SessionContext};
+use datafusion_cli::catalog::DynamicObjectStoreCatalog;
+use datafusion_cli::exec;
+use datafusion_cli::object_storage::instrumented::{
+    InstrumentedObjectStoreMode, InstrumentedObjectStoreRegistry,
+};
+use datafusion_cli::object_storage::{StdinCarriesCommands, is_stdin_location};
+use datafusion_cli::print_format::PrintFormat;
+use datafusion_cli::print_options::{MaxRows, PrintOptions};
+use datafusion_spark::SessionStateBuilderSpark;
+
+#[derive(Debug, Parser)]
+#[clap(version, about, long_about = None)]
+struct Args {
+    #[clap(
+        short = 'c',
+        long,
+        num_args = 0..,
+        help = "Execute the given command string(s), then exit",
+        value_parser(parse_command)
+    )]
+    command: Vec<String>,
+
+    #[clap(
+        short,
+        long,
+        num_args = 0..,
+        help = "Execute commands from file(s), then exit",
+        value_parser(parse_valid_file)
+    )]
+    file: Vec<String>,
+
+    #[clap(
+        short = 'r',
+        long,
+        num_args = 0..,
+        help = "Run the provided files on startup, before the REPL",
+        value_parser(parse_valid_file),
+        conflicts_with = "file"
+    )]
+    rc: Vec<String>,
+
+    #[clap(
+        short = 'p',
+        long,
+        help = "Path to your data, default to current directory",
+        value_parser(parse_valid_data_dir)
+    )]
+    data_path: Option<String>,
+
+    #[clap(
+        short = 'b',
+        long,
+        help = "The batch size of each query, or use DataFusion default",
+        value_parser(parse_batch_size)
+    )]
+    batch_size: Option<usize>,
+
+    #[clap(
+        long,
+        default_value_t = true,
+        action = ArgAction::Set,
+        help = "Evaluate with ANSI mode on, like Spark 4's default \
+                `spark.sql.ansi.enabled = true`. Pass `--ansi false` for 
Spark's legacy behavior"
+    )]
+    ansi: bool,
+
+    #[clap(long, value_enum, default_value_t = PrintFormat::Automatic)]
+    format: PrintFormat,
+
+    #[clap(
+        short,
+        long,
+        help = "Reduce printing other than the results and work quietly"
+    )]
+    quiet: bool,
+
+    #[clap(
+        long,
+        help = "The max number of rows to display for 'Table' 
format\n[possible values: numbers(0/10/...), inf(no limit)]",
+        default_value = "40"
+    )]
+    maxrows: MaxRows,
+
+    #[clap(long, help = "Enables console syntax highlighting")]
+    color: bool,
+}
+
+impl Args {
+    /// Without `-c` or `-f` the CLI enters the REPL, which reads SQL from 
stdin.
+    fn repl_mode(&self) -> bool {
+        self.command.is_empty() && self.file.is_empty()
+    }
+
+    /// Whether stdin carries the session's SQL, so it cannot also back a
+    /// `LOCATION '/dev/stdin'` table.
+    fn reads_sql_from_stdin(&self) -> bool {
+        self.repl_mode() || self.file.iter().any(|f| is_stdin_location(f))
+    }
+}
+
+#[tokio::main]
+async fn main() -> ExitCode {
+    if let Err(e) = main_inner().await {
+        println!("Error: {e}");
+        return ExitCode::FAILURE;
+    }
+    ExitCode::SUCCESS
+}
+
+async fn main_inner() -> Result<()> {
+    env_logger::init();
+    let args = Args::parse();
+
+    if !args.quiet {
+        println!(
+            "DataFusion Spark CLI v{} (ANSI mode {})",
+            env!("CARGO_PKG_VERSION"),
+            if args.ansi { "on" } else { "off" }
+        );
+    }
+
+    if let Some(path) = &args.data_path {

Review Comment:
   With `-p`, relative `-f` and `-r` paths can execute the wrong file or panic. 
Please absolutize script paths before changing directories, preserving 
`/dev/stdin`, and use `exec_from_lines` to handle file-open errors. Test both 
flags with conflicting filenames.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to