This is an automated email from the ASF dual-hosted git repository. github-bot pushed a commit to branch gh-readonly-queue/main/pr-2129-d78dbc97a1f65ce7ee353fea181a6c3bb15a5050 in repository https://gitbox.apache.org/repos/asf/datafusion-sqlparser-rs.git
commit 39418cfebbfe0c028d780614e8b60ef8f0c98ce7 Author: jnlt3 <[email protected]> AuthorDate: Thu Dec 18 15:06:55 2025 +0300 PostgreSQL Tokenization: Fix unexpected characters after question mark being silently ignored (#2129) --- src/tokenizer.rs | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2ae17cf4..8666563a 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -1717,7 +1717,7 @@ impl<'a> Tokenizer<'a> { } } Some('#') => self.consume_and_return(chars, Token::QuestionMarkSharp), - _ => self.consume_and_return(chars, Token::Question), + _ => Ok(Some(Token::Question)), } } '?' => { @@ -4147,4 +4147,23 @@ mod tests { panic!("Tokenizer should have failed on {sql}, but it succeeded with {tokens:?}"); } } + + #[test] + fn tokenize_question_mark() { + let dialect = PostgreSqlDialect {}; + let sql = "SELECT x ? y"; + let tokens = Tokenizer::new(&dialect, sql).tokenize().unwrap(); + compare( + tokens, + vec![ + Token::make_keyword("SELECT"), + Token::Whitespace(Whitespace::Space), + Token::make_word("x", None), + Token::Whitespace(Whitespace::Space), + Token::Question, + Token::Whitespace(Whitespace::Space), + Token::make_word("y", None), + ], + ) + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
