From f2d4087462571abd10825e665677efd4f98485f5 Mon Sep 17 00:00:00 2001 From: TcMits Date: Fri, 5 Sep 2025 12:58:28 +0700 Subject: [PATCH 1/5] support float without fractional part --- parser/src/lexer.rs | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index e644b8c95..604d14bcf 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -590,7 +590,7 @@ impl<'a> Lexer<'a> { self.eat_and_assert(|b| b == b'.'); match self.peek() { - Some(b) if b.is_ascii_digit() => { + Some(b) if b.is_ascii_digit() || b.eq_ignore_ascii_case(&b'e') => { self.eat_while_number_digit()?; match self.peek() { Some(b'e') | Some(b'E') => { @@ -1239,6 +1239,14 @@ mod tests { token_type: Some(TokenType::TK_ID), }, ), + // issue 2933 + ( + b"1.e5".as_slice(), + Token { + value: b"1.e5".as_slice(), // 'X' is not included in the value + token_type: Some(TokenType::TK_FLOAT), + }, + ), ]; for (input, expected) in test_cases { From 168f6dcbb56ac6230858a39c6eb503086b1b0bc1 Mon Sep 17 00:00:00 2001 From: TcMits Date: Fri, 5 Sep 2025 13:00:07 +0700 Subject: [PATCH 2/5] unrelated comments --- parser/src/lexer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 604d14bcf..1b64f0a73 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1243,7 +1243,7 @@ mod tests { ( b"1.e5".as_slice(), Token { - value: b"1.e5".as_slice(), // 'X' is not included in the value + value: b"1.e5".as_slice(), token_type: Some(TokenType::TK_FLOAT), }, ), From 4726ffaa748fa81a71fc5b150aa4f6fbbb86a748 Mon Sep 17 00:00:00 2001 From: TcMits Date: Fri, 5 Sep 2025 13:08:42 +0700 Subject: [PATCH 3/5] only check 'e' in eat_number --- parser/src/lexer.rs | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 1b64f0a73..decac0276 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -207,7 +207,7 @@ impl<'a> Iterator for Lexer<'a> { b'&' => Some(Ok(self.eat_one_token(TokenType::TK_BITAND))), b'~' => Some(Ok(self.eat_one_token(TokenType::TK_BITNOT))), b'\'' | b'"' | b'`' => Some(self.mark(|l| l.eat_lit_or_id())), - b'.' => Some(self.mark(|l| l.eat_dot_or_frac())), + b'.' => Some(self.mark(|l| l.eat_dot_or_frac(false))), b'0'..=b'9' => Some(self.mark(|l| l.eat_number())), b'[' => Some(self.mark(|l| l.eat_bracket())), b'?' | b'$' | b'@' | b'#' | b':' => Some(self.mark(|l| l.eat_var())), @@ -585,12 +585,14 @@ impl<'a> Lexer<'a> { }) } - fn eat_dot_or_frac(&mut self) -> Result> { + fn eat_dot_or_frac(&mut self, has_digit_prefix: bool) -> Result> { let start = self.offset; self.eat_and_assert(|b| b == b'.'); match self.peek() { - Some(b) if b.is_ascii_digit() || b.eq_ignore_ascii_case(&b'e') => { + Some(b) + if b.is_ascii_digit() || (has_digit_prefix && b.eq_ignore_ascii_case(&b'e')) => + { self.eat_while_number_digit()?; match self.peek() { Some(b'e') | Some(b'E') => { @@ -677,7 +679,7 @@ impl<'a> Lexer<'a> { self.eat_while_number_digit()?; match self.peek() { Some(b'.') => { - self.eat_dot_or_frac()?; + self.eat_dot_or_frac(true)?; Ok(Token { value: &self.input[start..self.offset], token_type: Some(TokenType::TK_FLOAT), @@ -1521,6 +1523,24 @@ mod tests { }, ], ), + // issue 2933 + ( + b"u.name".as_slice(), + vec![ + Token { + value: b"u".as_slice(), + token_type: Some(TokenType::TK_ID), + }, + Token { + value: b".".as_slice(), + token_type: Some(TokenType::TK_DOT), + }, + Token { + value: b"name".as_slice(), + token_type: Some(TokenType::TK_ID), + }, + ], + ), ]; for (input, expected_tokens) in test_cases { From 0f9ae5853c82bb22295375d3c28519ccd4abf4b5 Mon Sep 17 00:00:00 2001 From: TcMits Date: Fri, 5 Sep 2025 13:19:22 +0700 Subject: [PATCH 4/5] test --- parser/src/lexer.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index decac0276..4ea112164 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1525,7 +1525,7 @@ mod tests { ), // issue 2933 ( - b"u.name".as_slice(), + b"u.emai".as_slice(), vec![ Token { value: b"u".as_slice(), @@ -1536,7 +1536,7 @@ mod tests { token_type: Some(TokenType::TK_DOT), }, Token { - value: b"name".as_slice(), + value: b"email".as_slice(), token_type: Some(TokenType::TK_ID), }, ], From e725814ce813cdb55cc51dccd23a51b9ab756afb Mon Sep 17 00:00:00 2001 From: TcMits Date: Fri, 5 Sep 2025 13:38:33 +0700 Subject: [PATCH 5/5] fix test --- parser/src/lexer.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/parser/src/lexer.rs b/parser/src/lexer.rs index 4ea112164..5e9e809b8 100644 --- a/parser/src/lexer.rs +++ b/parser/src/lexer.rs @@ -1525,7 +1525,7 @@ mod tests { ), // issue 2933 ( - b"u.emai".as_slice(), + b"u.email".as_slice(), vec![ Token { value: b"u".as_slice(),