From b43c1544d4bcac5ee9f8231650e4f20f8bb8920f Mon Sep 17 00:00:00 2001
From: Krishna Vishal <krishnavishal2008@gmail.com>
Date: Mon, 20 Jan 2025 19:37:15 +0530
Subject: [PATCH] `Tokenizer::split()` function returns TK_ILLEGAL instead of
 Error.

---
 vendored/sqlite3-parser/src/lexer/sql/mod.rs | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/vendored/sqlite3-parser/src/lexer/sql/mod.rs b/vendored/sqlite3-parser/src/lexer/sql/mod.rs
index 112de2ad2..0b73590b3 100644
--- a/vendored/sqlite3-parser/src/lexer/sql/mod.rs
+++ b/vendored/sqlite3-parser/src/lexer/sql/mod.rs
@@ -489,11 +489,21 @@ impl Splitter for Tokenizer {
                     Ok(self.identifierish(data))
                 }
             }
-            _ => Err(Error::UnrecognizedToken(None, None)),
+            // Return TK_ILLEGAL
+            _ => handle_unrecognized(data),
         }
     }
 }
 
+fn handle_unrecognized(data: &[u8]) -> Result<(Option<Token<'_>>, usize), Error> {
+    let mut end = 1;
+    while end < data.len() && !data[end].is_ascii_whitespace() {
+        end += 1;
+    }
+
+    Ok((Some((&data[..end], TokenType::TK_ILLEGAL)), end))
+}
+
 fn literal(data: &[u8], quote: u8) -> Result<(Option<Token<'_>>, usize), Error> {
     debug_assert_eq!(data[0], quote);
     let tt = if quote == b'\'' { TK_STRING } else { TK_ID };