aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-02-23 00:45:37 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'refactor parser fmt' from Lâm Hoàng Phúc

Browse Source 
@penberg this PR try to clean up `turso_parser`'s`fmt` code.
- `get_table_name` and `get_column_name` should return None when
table/column does not exist.
```rust
/// Context to be used in ToSqlString
pub trait ToSqlContext {
    /// Given an id, get the table name
    /// First Option indicates whether the table exists
    ///
    /// Currently not considering aliases
    fn get_table_name(&self, _id: TableInternalId) -> Option<&str> {
        None
    }

    /// Given a table id and a column index, get the column name
    /// First Option indicates whether the column exists
    /// Second Option indicates whether the column has a name
    fn get_column_name(&self, _table_id: TableInternalId, _col_idx: usize) -> Option<Option<&str>> {
        None
    }

    // help function to handle missing table/column names
    fn get_table_and_column_names(
        &self,
        table_id: TableInternalId,
        col_idx: usize,
    ) -> (String, String) {
        let table_name = self
            .get_table_name(table_id)
            .map(|s| s.to_owned())
            .unwrap_or_else(|| format!("t{}", table_id.0));

        let column_name = self
            .get_column_name(table_id, col_idx)
            .map(|opt| {
                opt.map(|s| s.to_owned())
                    .unwrap_or_else(|| format!("c{col_idx}"))
            })
            .unwrap_or_else(|| format!("c{col_idx}"));

        (table_name, column_name)
    }
}
```
- remove `FmtTokenStream` because it is same as `WriteTokenStream `
- remove useless functions and simplify `ToTokens`
```rust
/// Generate token(s) from AST node
/// Also implements Display to make sure devs won't forget Display
pub trait ToTokens: Display {
    /// Send token(s) to the specified stream with context
    fn to_tokens<S: TokenStream + ?Sized, C: ToSqlContext>(
        &self,
        s: &mut S,
        context: &C,
    ) -> Result<(), S::Error>;

    // Return displayer representation with context
    fn displayer<'a, 'b, C: ToSqlContext>(&'b self, ctx: &'a C) -> SqlDisplayer<'a, 'b, C, Self>
    where
        Self: Sized,
    {
        SqlDisplayer::new(ctx, self)
    }
}
```

Closes #2748
This commit is contained in:
Pekka Enberg
2025-09-02 18:35:43 +03:00
committed by GitHub
parent 2addeb5a9f 33a04fbaf7
commit 1de647758f
16 changed files with 576 additions and 576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
core/incremental/view.rs
View File

@@ -140,14 +140,7 @@ impl IncrementalView {
Parser::new(sql.as_bytes()).next_cmd()
{
// Compare the SELECT statements as SQL strings
use turso_parser::ast::fmt::ToTokens;
// Format both SELECT statements and compare
if let (Ok(current_sql), Ok(provided_sql)) =
(self.select_stmt.format(), select.format())
{
return current_sql == provided_sql;
}
return self.select_stmt == select;
}
false
}

7
core/translate/alter.rs
View File

@@ -1,8 +1,5 @@
use std::sync::Arc;
use turso_parser::{
ast::{self, fmt::ToTokens as _},
parser::Parser,
};
use turso_parser::{ast, parser::Parser};
use crate::{
function::{AlterTableFunc, Func},
@@ -413,7 +410,7 @@ pub fn translate_alter_table(
program.emit_string8_new_reg(from.to_string());
program.mark_last_insn_constant();
program.emit_string8_new_reg(definition.format().unwrap());
program.emit_string8_new_reg(definition.to_string());
program.mark_last_insn_constant();
let out = program.alloc_registers(sqlite_schema_column_len);

68
core/translate/display.rs
View File

@@ -3,7 +3,7 @@ use std::fmt::{Display, Formatter};
use turso_parser::{
ast::{
self,
fmt::{ToSqlContext, ToTokens, TokenStream},
fmt::{BlankContext, ToSqlContext, ToTokens, TokenStream},
SortOrder, TableInternalId,
},
token::TokenType,
@@ -235,44 +235,40 @@ pub struct PlanContext<'a>(pub &'a [&'a TableReferences]);
// Definitely not perfect yet
impl ToSqlContext for PlanContext<'_> {
fn get_column_name(&self, table_id: TableInternalId, col_idx: usize) -> String {
fn get_column_name(&self, table_id: TableInternalId, col_idx: usize) -> Option<Option<&str>> {
let table = self
.0
.iter()
.find_map(|table_ref| table_ref.find_table_by_internal_id(table_id))
.unwrap();
.find_map(|table_ref| table_ref.find_table_by_internal_id(table_id))?;
let cols = table.columns();
match cols.get(col_idx).unwrap().name.as_ref() {
None => format!("{col_idx}"),
Some(n) => n.to_string(),
}
cols.get(col_idx)
.map(|col| col.name.as_ref().map(|name| name.as_ref()))
}
fn get_table_name(&self, id: TableInternalId) -> &str {
fn get_table_name(&self, id: TableInternalId) -> Option<&str> {
let table_ref = self
.0
.iter()
.find(|table_ref| table_ref.find_table_by_internal_id(id).is_some())
.unwrap();
.find(|table_ref| table_ref.find_table_by_internal_id(id).is_some())?;
let joined_table = table_ref.find_joined_table_by_internal_id(id);
let outer_query = table_ref.find_outer_query_ref_by_internal_id(id);
match (joined_table, outer_query) {
(Some(table), None) => &table.identifier,
(None, Some(table)) => &table.identifier,
(Some(table), None) => Some(&table.identifier),
(None, Some(table)) => Some(&table.identifier),
_ => unreachable!(),
}
}
}
impl ToTokens for Plan {
fn to_tokens_with_context<S: TokenStream + ?Sized, C: ToSqlContext>(
fn to_tokens<S: TokenStream + ?Sized, C: ToSqlContext>(
&self,
s: &mut S,
context: &C,
) -> Result<(), S::Error> {
match self {
Self::Select(select) => {
select.to_tokens_with_context(s, &PlanContext(&[&select.table_references]))?;
select.to_tokens(s, &PlanContext(&[&select.table_references]))?;
}
Self::CompoundSelect {
left,
@@ -289,11 +285,11 @@ impl ToTokens for Plan {
let context = &PlanContext(all_refs.as_slice());
for (plan, operator) in left {
plan.to_tokens_with_context(s, context)?;
operator.to_tokens_with_context(s, context)?;
plan.to_tokens(s, context)?;
operator.to_tokens(s, context)?;
}
right_most.to_tokens_with_context(s, context)?;
right_most.to_tokens(s, context)?;
if let Some(order_by) = order_by {
s.append(TokenType::TK_ORDER, None)?;
@@ -319,16 +315,22 @@ impl ToTokens for Plan {
s.append(TokenType::TK_FLOAT, Some(&offset.to_string()))?;
}
}
Self::Delete(delete) => delete.to_tokens_with_context(s, context)?,
Self::Update(update) => update.to_tokens_with_context(s, context)?,
Self::Delete(delete) => delete.to_tokens(s, context)?,
Self::Update(update) => update.to_tokens(s, context)?,
}
Ok(())
}
}
impl Display for JoinedTable {
fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
self.displayer(&BlankContext).fmt(f)
}
}
impl ToTokens for JoinedTable {
fn to_tokens_with_context<S: TokenStream + ?Sized, C: ToSqlContext>(
fn to_tokens<S: TokenStream + ?Sized, C: ToSqlContext>(
&self,
s: &mut S,
_context: &C,
@@ -345,7 +347,7 @@ impl ToTokens for JoinedTable {
Table::FromClauseSubquery(from_clause_subquery) => {
s.append(TokenType::TK_LP, None)?;
// Could possibly merge the contexts together here
from_clause_subquery.plan.to_tokens_with_context(
from_clause_subquery.plan.to_tokens(
s,
&PlanContext(&[&from_clause_subquery.plan.table_references]),
)?;
@@ -362,7 +364,7 @@ impl ToTokens for JoinedTable {
// TODO: currently cannot print the original CTE as it is optimized into a subquery
impl ToTokens for SelectPlan {
fn to_tokens_with_context<S: TokenStream + ?Sized, C: ToSqlContext>(
fn to_tokens<S: TokenStream + ?Sized, C: ToSqlContext>(
&self,
s: &mut S,
context: &C,
@@ -374,7 +376,7 @@ impl ToTokens for SelectPlan {
.map(|values| values.iter().map(|v| Box::from(v.clone())).collect())
.collect(),
)
.to_tokens_with_context(s, context)?;
.to_tokens(s, context)?;
} else {
s.append(TokenType::TK_SELECT, None)?;
if self.distinctness.is_distinct() {
@@ -386,7 +388,7 @@ impl ToTokens for SelectPlan {
s.append(TokenType::TK_COMMA, None)?;
}
expr.to_tokens_with_context(s, context)?;
expr.to_tokens(s, context)?;
if let Some(alias) = alias {
s.append(TokenType::TK_AS, None)?;
s.append(TokenType::TK_ID, Some(alias))?;
@@ -403,7 +405,7 @@ impl ToTokens for SelectPlan {
}
let table_ref = self.joined_tables().get(order.original_idx).unwrap();
table_ref.to_tokens_with_context(s, context)?;
table_ref.to_tokens(s, context)?;
}
if !self.where_clause.is_empty() {
@@ -418,7 +420,7 @@ impl ToTokens for SelectPlan {
if i != 0 {
s.append(TokenType::TK_AND, None)?;
}
expr.to_tokens_with_context(s, context)?;
expr.to_tokens(s, context)?;
}
}
@@ -436,7 +438,7 @@ impl ToTokens for SelectPlan {
if i != 0 {
s.append(TokenType::TK_AND, None)?;
}
expr.to_tokens_with_context(s, context)?;
expr.to_tokens(s, context)?;
}
}
}
@@ -471,7 +473,7 @@ impl ToTokens for SelectPlan {
}
impl ToTokens for DeletePlan {
fn to_tokens_with_context<S: TokenStream + ?Sized, C: ToSqlContext>(
fn to_tokens<S: TokenStream + ?Sized, C: ToSqlContext>(
&self,
s: &mut S,
_: &C,
@@ -500,7 +502,7 @@ impl ToTokens for DeletePlan {
if i != 0 {
s.append(TokenType::TK_AND, None)?;
}
expr.to_tokens_with_context(s, context)?;
expr.to_tokens(s, context)?;
}
}
@@ -533,7 +535,7 @@ impl ToTokens for DeletePlan {
}
impl ToTokens for UpdatePlan {
fn to_tokens_with_context<S: TokenStream + ?Sized, C: ToSqlContext>(
fn to_tokens<S: TokenStream + ?Sized, C: ToSqlContext>(
&self,
s: &mut S,
_: &C,
@@ -578,10 +580,10 @@ impl ToTokens for UpdatePlan {
.map(|where_clause| where_clause.expr.clone());
iter.next()
.expect("should not be empty")
.to_tokens_with_context(s, context)?;
.to_tokens(s, context)?;
for expr in iter {
s.append(TokenType::TK_AND, None)?;
expr.to_tokens_with_context(s, context)?;
expr.to_tokens(s, context)?;
}
}

28
core/translate/main_loop.rs
View File

@@ -1,4 +1,4 @@
use turso_parser::ast::SortOrder;
use turso_parser::ast::{fmt::ToTokens, SortOrder};
use std::sync::Arc;
@@ -20,6 +20,7 @@ use crate::{
use super::{
aggregation::{translate_aggregation_step, AggArgumentSource},
display::PlanContext,
emitter::{OperationMode, TranslateCtx},
expr::{
translate_condition_expr, translate_expr, translate_expr_no_constant_opt,
@@ -77,12 +78,17 @@ pub fn init_distinct(program: &mut ProgramBuilder, plan: &SelectPlan) -> Distinc
.result_columns
.iter()
.enumerate()
.map(|(i, col)| IndexColumn {
name: col.expr.to_string(),
order: SortOrder::Asc,
pos_in_table: i,
collation: None, // FIXME: this should be determined based on the result column expression!
default: None, // FIXME: this should be determined based on the result column expression!
.map(|(i, col)| {
IndexColumn {
name: col
.expr
.displayer(&PlanContext(&[&plan.table_references]))
.to_string(),
order: SortOrder::Asc,
pos_in_table: i,
collation: None, // FIXME: this should be determined based on the result column expression!
default: None, // FIXME: this should be determined based on the result column expression!
}
})
.collect(),
unique: false,
@@ -141,14 +147,18 @@ pub fn init_loop(
agg.args.len() == 1,
"DISTINCT aggregate functions must have exactly one argument"
);
let index_name = format!("distinct_agg_{}_{}", i, agg.args[0]);
let index_name = format!(
"distinct_agg_{}_{}",
i,
agg.args[0].displayer(&PlanContext(&[tables]))
);
let index = Arc::new(Index {
name: index_name.clone(),
table_name: String::new(),
ephemeral: true,
root_page: 0,
columns: vec![IndexColumn {
name: agg.args[0].to_string(),
name: agg.args[0].displayer(&PlanContext(&[tables])).to_string(),
order: SortOrder::Asc,
pos_in_table: 0,
collation: None, // FIXME: this should be inferred from the expression

8
core/translate/optimizer/mod.rs
View File

@@ -8,7 +8,7 @@ use join::{compute_best_join_order, BestJoinOrderResult};
use lift_common_subexpressions::lift_common_subexpressions_from_binary_or_terms;
use order::{compute_order_target, plan_satisfies_order_target, EliminatesSortBy};
use turso_ext::{ConstraintInfo, ConstraintUsage};
use turso_parser::ast::{self, fmt::ToTokens as _, Expr, SortOrder};
use turso_parser::ast::{self, Expr, SortOrder};
use crate::{
parameters::PARAM_PREFIX,
@@ -52,11 +52,7 @@ pub fn optimize_plan(plan: &mut Plan, schema: &Schema) -> Result<()> {
}
}
// When debug tracing is enabled, print the optimized plan as a SQL string for debugging
tracing::debug!(
plan_sql = plan
.format_with_context(&crate::translate::display::PlanContext(&[]))
.unwrap()
);
tracing::debug!(plan_sql = plan.to_string());
Ok(())
}

10
core/translate/schema.rs
View File

@@ -28,7 +28,6 @@ use crate::SymbolTable;
use crate::{bail_parse_error, Result};
use turso_ext::VTabKind;
use turso_parser::ast::fmt::ToTokens;
#[allow(clippy::too_many_arguments)]
pub fn translate_create_table(
@@ -509,14 +508,7 @@ enum PrimaryKeyDefinitionType<'a> {
fn create_table_body_to_str(tbl_name: &ast::QualifiedName, body: &ast::CreateTableBody) -> String {
let mut sql = String::new();
sql.push_str(
format!(
"CREATE TABLE {} {}",
tbl_name.name.as_str(),
body.format().unwrap()
)
.as_str(),
);
sql.push_str(format!("CREATE TABLE {} {}", tbl_name.name.as_str(), body).as_str());
match body {
ast::CreateTableBody::ColumnsAndConstraints {
columns: _,

14
core/translate/view.rs
View File

@@ -6,7 +6,7 @@ use crate::vdbe::builder::{CursorType, ProgramBuilder};
use crate::vdbe::insn::{CmpInsFlags, Cookie, Insn};
use crate::{Connection, Result, SymbolTable};
use std::sync::Arc;
use turso_parser::ast::{self, fmt::ToTokens};
use turso_parser::ast;
/// Common logic for creating views (both regular and materialized)
fn emit_create_view_program(
@@ -109,11 +109,7 @@ pub fn translate_create_materialized_view(
}
fn create_materialized_view_to_str(view_name: &str, select_stmt: &ast::Select) -> String {
format!(
"CREATE MATERIALIZED VIEW {} AS {}",
view_name,
select_stmt.format().unwrap()
)
format!("CREATE MATERIALIZED VIEW {view_name} AS {select_stmt}")
}
pub fn translate_create_view(
@@ -148,11 +144,7 @@ pub fn translate_create_view(
}
fn create_view_to_str(view_name: &str, select_stmt: &ast::Select) -> String {
format!(
"CREATE VIEW {} AS {}",
view_name,
select_stmt.format().unwrap()
)
format!("CREATE VIEW {view_name} AS {select_stmt}")
}
pub fn translate_drop_view(

2
core/util.rs
View File

@@ -1261,7 +1261,7 @@ pub fn extract_view_columns(select_stmt: &ast::Select, schema: &Schema) -> Vec<C
.or_else(|| extract_column_name_from_expr(expr))
.unwrap_or_else(|| {
// If we can't extract a simple column name, use the expression itself
expr.format().unwrap_or_else(|_| format!("column_{i}"))
expr.to_string()
});
columns.push(Column {
name: Some(name),

13
core/vdbe/execute.rs
View File

@@ -74,7 +74,6 @@ use super::{
use parking_lot::RwLock;
use rand::{thread_rng, Rng};
use turso_parser::ast;
use turso_parser::ast::fmt::ToTokens;
use turso_parser::parser::Parser;
use super::{
@@ -4880,8 +4879,7 @@ pub fn op_function(
columns,
where_clause,
}
.format()
.unwrap(),
.to_string(),
)
}
ast::Stmt::CreateTable {
@@ -4907,8 +4905,7 @@ pub fn op_function(
if_not_exists,
body,
}
.format()
.unwrap(),
.to_string(),
)
}
_ => todo!(),
@@ -4990,8 +4987,7 @@ pub fn op_function(
idx_name,
where_clause,
}
.format()
.unwrap(),
.to_string(),
)
}
ast::Stmt::CreateTable {
@@ -5037,8 +5033,7 @@ pub fn op_function(
temporary,
if_not_exists,
}
.format()
.unwrap(),
.to_string(),
)
}
_ => todo!(),

15
parser/src/ast.rs
View File

@@ -3,8 +3,6 @@ pub mod fmt;
use strum_macros::{EnumIter, EnumString};
use crate::ast::fmt::ToTokens;
/// `?` or `$` Prepared statement arg placeholder(s)
#[derive(Default)]
pub struct ParameterInfo {
@@ -931,13 +929,6 @@ pub struct QualifiedName {
pub alias: Option<Name>, // FIXME restrict alias usage (fullname vs xfullname)
}
impl std::fmt::Display for QualifiedName {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
use fmt::ToTokens as _;
self.to_fmt(f)
}
}
impl QualifiedName {
/// Constructor
pub fn single(name: Name) -> Self {
@@ -1155,12 +1146,6 @@ pub enum SortOrder {
Desc,
}
impl core::fmt::Display for SortOrder {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.to_fmt(f)
}
}
/// `NULLS FIRST` or `NULLS LAST`
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]

921
parser/src/ast/fmt.rs
View File

File diff suppressed because it is too large Load Diff

13
parser/src/parser.rs
View File

@@ -11483,14 +11483,23 @@ mod tests {
];
for (input, expected) in test_cases {
println!("Testing input: {:?}", from_bytes(input));
let input_str = from_bytes(input);
let parser = Parser::new(input);
let mut results = Vec::new();
for cmd in parser {
results.push(cmd.unwrap());
}
assert_eq!(results, expected, "Input: {input:?}");
assert_eq!(results, expected, "Input: {input_str:?}");
// to_string tests
for (i, r) in results.iter().enumerate() {
let rstring = r.to_string();
// put new string into parser again
let result = Parser::new(rstring.as_bytes()).next().unwrap().unwrap();
let expected = &expected[i];
assert_eq!(result, expected.clone(), "Input: {rstring:?}");
}
}
}
}

6
simulator/model/mod.rs
View File

@@ -5,14 +5,14 @@ use itertools::Itertools;
use serde::{Deserialize, Serialize};
use sql_generation::model::{
query::{
Create, CreateIndex, Delete, Drop, EmptyContext, Insert, Select,
Create, CreateIndex, Delete, Drop, Insert, Select,
select::{CompoundOperator, FromClause, ResultColumn, SelectInner},
transaction::{Begin, Commit, Rollback},
update::Update,
},
table::{JoinTable, JoinType, SimValue, Table, TableContext},
};
use turso_parser::ast::{Distinctness, fmt::ToTokens};
use turso_parser::ast::Distinctness;
use crate::{generation::Shadow, runner::env::SimulatorTables};
@@ -307,7 +307,7 @@ impl Shadow for SelectInner {
} else {
return Err(anyhow::anyhow!(
"Failed to evaluate expression in free select ({})",
expr.0.format_with_context(&EmptyContext {}).unwrap()
expr.0
));
}
}

18
sql_generation/model/query/mod.rs
View File

@@ -4,7 +4,6 @@ pub use delete::Delete;
pub use drop::Drop;
pub use insert::Insert;
pub use select::Select;
use turso_parser::ast::fmt::ToSqlContext;
pub mod create;
pub mod create_index;
@@ -15,20 +14,3 @@ pub mod predicate;
pub mod select;
pub mod transaction;
pub mod update;
/// Used to print sql strings that already have all the context it needs
pub struct EmptyContext;
impl ToSqlContext for EmptyContext {
fn get_column_name(
&self,
_table_id: turso_parser::ast::TableInternalId,
_col_idx: usize,
) -> String {
unreachable!()
}
fn get_table_name(&self, _id: turso_parser::ast::TableInternalId) -> &str {
unreachable!()
}
}

7
sql_generation/model/query/predicate.rs
View File

@@ -1,7 +1,10 @@
use std::fmt::Display;
use serde::{Deserialize, Serialize};
use turso_parser::ast::{self, fmt::ToTokens};
use turso_parser::ast::{
self,
fmt::{BlankContext, ToTokens},
};
use crate::model::table::{SimValue, Table, TableContext};
@@ -142,6 +145,6 @@ pub fn expr_to_value<T: TableContext>(
impl Display for Predicate {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.0.to_fmt(f)
self.0.displayer(&BlankContext).fmt(f)
}
}

13
sql_generation/model/query/select.rs
View File

@@ -3,13 +3,14 @@ use std::{collections::HashSet, fmt::Display};
pub use ast::Distinctness;
use itertools::Itertools;
use serde::{Deserialize, Serialize};
use turso_parser::ast::{self, fmt::ToTokens, SortOrder};
use crate::model::{
query::EmptyContext,
table::{JoinTable, JoinType, JoinedTable, Table},
use turso_parser::ast::{
self,
fmt::{BlankContext, ToTokens},
SortOrder,
};
use crate::model::table::{JoinTable, JoinType, JoinedTable, Table};
use super::predicate::Predicate;
/// `SELECT` or `RETURNING` result column
@@ -366,7 +367,7 @@ impl Select {
impl Display for Select {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
self.to_sql_ast().to_fmt_with_context(f, &EmptyContext {})
self.to_sql_ast().displayer(&BlankContext).fmt(f)
}
}