Tokenizer::split() function returns TK_ILLEGAL instead of Error.

vendored/sqlite3-parser/src/lexer/sql/mod.rs

@@ -489,11 +489,21 @@ impl Splitter for Tokenizer {
                     Ok(self.identifierish(data))
                 }
             }
-            _ => Err(Error::UnrecognizedToken(None, None)),
+            // Return TK_ILLEGAL
+            _ => handle_unrecognized(data),
         }
     }
 }
 
+fn handle_unrecognized(data: &[u8]) -> Result<(Option<Token<'_>>, usize), Error> {
+    let mut end = 1;
+    while end < data.len() && !data[end].is_ascii_whitespace() {
+        end += 1;
+    }
+
+    Ok((Some((&data[..end], TokenType::TK_ILLEGAL)), end))
+}
+
 fn literal(data: &[u8], quote: u8) -> Result<(Option<Token<'_>>, usize), Error> {
     debug_assert_eq!(data[0], quote);
     let tt = if quote == b'\'' { TK_STRING } else { TK_ID };