From 05f0ee6a72df1f849cd6da8dde9841427c6aafd8 Mon Sep 17 00:00:00 2001 From: Nikita Sivukhin Date: Mon, 27 Oct 2025 17:00:26 +0400 Subject: [PATCH] add more integration in order to properly skip backing_btree index_method --- core/incremental/operator.rs | 1 + core/io/generic.rs | 4 +- core/schema.rs | 79 ++++++++++++++++++++++++++----- core/storage/btree.rs | 2 + core/translate/compound_select.rs | 1 + core/translate/emitter.rs | 30 +++++------- core/translate/index.rs | 7 +-- core/translate/main_loop.rs | 72 +++++++++++++--------------- core/translate/optimizer/join.rs | 8 ++++ core/translate/optimizer/mod.rs | 1 + core/translate/order_by.rs | 1 + core/translate/plan.rs | 3 ++ core/util.rs | 2 +- 13 files changed, 135 insertions(+), 76 deletions(-) diff --git a/core/incremental/operator.rs b/core/incremental/operator.rs index 764a59167..d93947dcc 100644 --- a/core/incremental/operator.rs +++ b/core/incremental/operator.rs @@ -70,6 +70,7 @@ pub fn create_dbsp_state_index(root_page: i64) -> Index { ephemeral: false, has_rowid: true, where_clause: None, + index_method: None, } } diff --git a/core/io/generic.rs b/core/io/generic.rs index b465a24cb..caab217bb 100644 --- a/core/io/generic.rs +++ b/core/io/generic.rs @@ -1,6 +1,4 @@ -use crate::{ - io::clock::DefaultClock, Clock, Completion, File, Instant, LimboError, OpenFlags, Result, IO, -}; +use crate::{io::clock::DefaultClock, Clock, Completion, File, Instant, OpenFlags, Result, IO}; use parking_lot::RwLock; use std::io::{Read, Seek, Write}; use std::sync::Arc; diff --git a/core/schema.rs b/core/schema.rs index d256d9cac..b8a08546c 100644 --- a/core/schema.rs +++ b/core/schema.rs @@ -1,9 +1,10 @@ use crate::function::Func; use crate::incremental::view::IncrementalView; +use crate::index_method::{IndexMethodAttachment, IndexMethodConfiguration}; use crate::translate::expr::{ bind_and_rewrite_expr, walk_expr, BindingBehavior, ParamState, WalkControl, }; -use crate::translate::index::resolve_sorted_columns; +use crate::translate::index::{resolve_index_method_parameters, resolve_sorted_columns}; use crate::translate::planner::ROWID_STRS; use parking_lot::RwLock; @@ -368,6 +369,7 @@ impl Schema { .get(&name) .map(|v| v.iter()) .unwrap_or_default() + .filter(|i| !i.is_backing_btree_index()) } pub fn get_index(&self, table_name: &str, index_name: &str) -> Option<&Arc> { @@ -485,7 +487,7 @@ impl Schema { pager.end_read_tx(); - self.populate_indices(from_sql_indexes, automatic_indices)?; + self.populate_indices(syms, from_sql_indexes, automatic_indices)?; self.populate_materialized_views( materialized_view_info, @@ -501,6 +503,7 @@ impl Schema { /// automatic_indices: indices created automatically for primary key and unique constraints pub fn populate_indices( &mut self, + syms: &SymbolTable, from_sql_indexes: Vec, automatic_indices: std::collections::HashMap>, ) -> Result<()> { @@ -512,6 +515,7 @@ impl Schema { .get_btree_table(&unparsed_sql_from_index.table_name) .unwrap(); let index = Index::from_sql( + syms, &unparsed_sql_from_index.sql, unparsed_sql_from_index.root_page, table.as_ref(), @@ -2419,6 +2423,7 @@ pub struct Index { /// and SELECT DISTINCT ephemeral indexes will not have a rowid. pub has_rowid: bool, pub where_clause: Option>, + pub index_method: Option>, } #[allow(dead_code)] @@ -2437,7 +2442,12 @@ pub struct IndexColumn { } impl Index { - pub fn from_sql(sql: &str, root_page: i64, table: &BTreeTable) -> Result { + pub fn from_sql( + syms: &SymbolTable, + sql: &str, + root_page: i64, + table: &BTreeTable, + ) -> Result { let mut parser = Parser::new(sql.as_bytes()); let cmd = parser.next_cmd()?; match cmd { @@ -2447,25 +2457,66 @@ impl Index { columns, unique, where_clause, + using, + with_clause, .. })) => { let index_name = normalize_ident(idx_name.name.as_str()); let index_columns = resolve_sorted_columns(table, &columns)?; - Ok(Index { - name: index_name, - table_name: normalize_ident(tbl_name.as_str()), - root_page, - columns: index_columns, - unique, - ephemeral: false, - has_rowid: table.has_rowid, - where_clause, - }) + if let Some(using) = using { + if where_clause.is_some() { + bail_parse_error!("custom index module do not support partial indices"); + } + if unique { + bail_parse_error!("custom index module do not support UNIQUE indices"); + } + let parameters = resolve_index_method_parameters(with_clause)?; + let Some(module) = syms.index_methods.get(using.as_str()) else { + bail_parse_error!("unknown module name: '{}'", using); + }; + let configuration = IndexMethodConfiguration { + table_name: table.name.clone(), + index_name: index_name.clone(), + columns: index_columns.clone(), + parameters, + }; + let descriptor = module.attach(&configuration)?; + Ok(Index { + name: index_name, + table_name: normalize_ident(tbl_name.as_str()), + root_page, + columns: index_columns, + unique: false, + ephemeral: false, + has_rowid: table.has_rowid, + where_clause: None, + index_method: Some(descriptor), + }) + } else { + Ok(Index { + name: index_name, + table_name: normalize_ident(tbl_name.as_str()), + root_page, + columns: index_columns, + unique, + ephemeral: false, + has_rowid: table.has_rowid, + where_clause, + index_method: None, + }) + } } _ => todo!("Expected create index statement"), } } + /// check if this is special backing_btree index created and managed by custom index_method + pub fn is_backing_btree_index(&self) -> bool { + self.index_method + .as_ref() + .is_some_and(|x| x.definition().backing_btree) + } + pub fn automatic_from_primary_key( table: &BTreeTable, auto_index: (String, i64), // name, root_page @@ -2505,6 +2556,7 @@ impl Index { ephemeral: false, has_rowid: table.has_rowid, where_clause: None, + index_method: None, }) } @@ -2542,6 +2594,7 @@ impl Index { ephemeral: false, has_rowid: table.has_rowid, where_clause: None, + index_method: None, }) } diff --git a/core/storage/btree.rs b/core/storage/btree.rs index fff63726f..bebe35a71 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -8553,6 +8553,7 @@ mod tests { unique: false, ephemeral: false, has_rowid: false, + index_method: None, }; let num_columns = index_def.columns.len(); let mut cursor = @@ -8712,6 +8713,7 @@ mod tests { unique: false, ephemeral: false, has_rowid: false, + index_method: None, }; let mut cursor = BTreeCursor::new_index(pager.clone(), index_root_page, &index_def, 1); diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 619d07585..783d58b66 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -432,6 +432,7 @@ fn create_dedupe_index( unique: false, has_rowid: false, where_clause: None, + index_method: None, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(dedupe_index.clone())); program.emit_insn(Insn::OpenEphemeral { diff --git a/core/translate/emitter.rs b/core/translate/emitter.rs index ff22d1035..0dd7d3a6e 100644 --- a/core/translate/emitter.rs +++ b/core/translate/emitter.rs @@ -719,30 +719,24 @@ fn emit_delete_insns( }); } else { // Delete from all indexes before deleting from the main table. - let indexes = t_ctx.resolver.schema.indexes.get(table_name); + let indexes = t_ctx.resolver.schema.get_indices(table_name); // Get the index that is being used to iterate the deletion loop, if there is one. let iteration_index = unsafe { &*table_reference }.op.index(); // Get all indexes that are not the iteration index. let other_indexes = indexes - .map(|indexes| { - indexes - .iter() - .filter(|index| { - iteration_index - .as_ref() - .is_none_or(|it_idx| !Arc::ptr_eq(it_idx, index)) - }) - .map(|index| { - ( - index.clone(), - program - .resolve_cursor_id(&CursorKey::index(internal_id, index.clone())), - ) - }) - .collect::>() + .filter(|index| { + iteration_index + .as_ref() + .is_none_or(|it_idx| !Arc::ptr_eq(it_idx, index)) }) - .unwrap_or_default(); + .map(|index| { + ( + index.clone(), + program.resolve_cursor_id(&CursorKey::index(internal_id, index.clone())), + ) + }) + .collect::>(); for (index, index_cursor_id) in other_indexes { let skip_delete_label = if index.where_clause.is_some() { diff --git a/core/translate/index.rs b/core/translate/index.rs index fc4c22176..889827d3e 100644 --- a/core/translate/index.rs +++ b/core/translate/index.rs @@ -119,7 +119,7 @@ pub fn translate_create_index( ); } - let mut module = None; + let mut index_method = None; if let Some(using) = &using { let index_modules = &resolver.symbol_table.index_methods; let using = using.as_str(); @@ -129,7 +129,7 @@ pub fn translate_create_index( } if let Some(index_module) = index_module { let parameters = resolve_index_method_parameters(with_clause)?; - module = Some(index_module.attach(&IndexMethodConfiguration { + index_method = Some(index_module.attach(&IndexMethodConfiguration { table_name: tbl.name.clone(), index_name: idx_name.clone(), columns: columns.clone(), @@ -148,6 +148,7 @@ pub fn translate_create_index( // store the *original* where clause, because we need to rewrite it // before translating, and it cannot reference a table alias where_clause: where_clause.clone(), + index_method: index_method.clone(), }); if !idx.validate_where_expr(table) { @@ -225,7 +226,7 @@ pub fn translate_create_index( Some(sql), )?; - if module.is_none() { + if index_method.is_none() { // determine the order of the columns in the index for the sorter let order = idx.columns.iter().map(|c| c.order).collect(); // open the sorter and the pseudo table diff --git a/core/translate/main_loop.rs b/core/translate/main_loop.rs index 32afc1c21..11c5b5e2b 100644 --- a/core/translate/main_loop.rs +++ b/core/translate/main_loop.rs @@ -99,6 +99,7 @@ pub fn init_distinct(program: &mut ProgramBuilder, plan: &SelectPlan) -> Result< unique: false, has_rowid: false, where_clause: None, + index_method: None, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); let ctx = DistinctCtx { @@ -173,6 +174,7 @@ pub fn init_loop( has_rowid: false, unique: false, where_clause: None, + index_method: None, }); let cursor_id = program.alloc_cursor_id(CursorType::BTreeIndex(index.clone())); if group_by.is_none() { @@ -240,25 +242,23 @@ pub fn init_loop( }); } // For delete, we need to open all the other indexes too for writing - if let Some(indexes) = t_ctx.resolver.schema.indexes.get(&btree.name) { - for index in indexes { - if table - .op - .index() - .is_some_and(|table_index| table_index.name == index.name) - { - continue; - } - let cursor_id = program.alloc_cursor_id_keyed( - CursorKey::index(table.internal_id, index.clone()), - CursorType::BTreeIndex(index.clone()), - ); - program.emit_insn(Insn::OpenWrite { - cursor_id, - root_page: index.root_page.into(), - db: table.database_id, - }); + for index in t_ctx.resolver.schema.get_indices(&btree.name) { + if table + .op + .index() + .is_some_and(|table_index| table_index.name == index.name) + { + continue; } + let cursor_id = program.alloc_cursor_id_keyed( + CursorKey::index(table.internal_id, index.clone()), + CursorType::BTreeIndex(index.clone()), + ); + program.emit_insn(Insn::OpenWrite { + cursor_id, + root_page: index.root_page.into(), + db: table.database_id, + }); } } (OperationMode::UPDATE(update_mode), Table::BTree(btree)) => { @@ -335,27 +335,23 @@ pub fn init_loop( // For DELETE, we need to open all the indexes for writing // UPDATE opens these in emit_program_for_update() separately if matches!(mode, OperationMode::DELETE) { - if let Some(indexes) = - t_ctx.resolver.schema.indexes.get(table.table.get_name()) - { - for index in indexes { - if table - .op - .index() - .is_some_and(|table_index| table_index.name == index.name) - { - continue; - } - let cursor_id = program.alloc_cursor_id_keyed( - CursorKey::index(table.internal_id, index.clone()), - CursorType::BTreeIndex(index.clone()), - ); - program.emit_insn(Insn::OpenWrite { - cursor_id, - root_page: index.root_page.into(), - db: table.database_id, - }); + for index in t_ctx.resolver.schema.get_indices(table.table.get_name()) { + if table + .op + .index() + .is_some_and(|table_index| table_index.name == index.name) + { + continue; } + let cursor_id = program.alloc_cursor_id_keyed( + CursorKey::index(table.internal_id, index.clone()), + CursorType::BTreeIndex(index.clone()), + ); + program.emit_insn(Insn::OpenWrite { + cursor_id, + root_page: index.root_page.into(), + db: table.database_id, + }); } } } diff --git a/core/translate/optimizer/join.rs b/core/translate/optimizer/join.rs index 79b80174a..d7d81e942 100644 --- a/core/translate/optimizer/join.rs +++ b/core/translate/optimizer/join.rs @@ -675,6 +675,7 @@ mod tests { ephemeral: false, root_page: 1, has_rowid: true, + index_method: None, }); available_indexes.insert("test_table".to_string(), VecDeque::from([index])); @@ -744,6 +745,7 @@ mod tests { ephemeral: false, root_page: 1, has_rowid: true, + index_method: None, }); available_indexes.insert("table1".to_string(), VecDeque::from([index1])); @@ -861,6 +863,7 @@ mod tests { ephemeral: false, root_page: 1, has_rowid: true, + index_method: None, }); available_indexes.insert(table_name.to_string(), VecDeque::from([index])); }); @@ -879,6 +882,7 @@ mod tests { ephemeral: false, root_page: 1, has_rowid: true, + index_method: None, }); let order_id_idx = Arc::new(Index { name: "order_items_order_id_idx".to_string(), @@ -895,6 +899,7 @@ mod tests { ephemeral: false, root_page: 1, has_rowid: true, + index_method: None, }); available_indexes @@ -1318,6 +1323,7 @@ mod tests { root_page: 2, ephemeral: false, has_rowid: true, + index_method: None, }); let mut available_indexes = HashMap::new(); @@ -1412,6 +1418,7 @@ mod tests { root_page: 2, ephemeral: false, has_rowid: true, + index_method: None, }); available_indexes.insert("t1".to_string(), VecDeque::from([index])); @@ -1524,6 +1531,7 @@ mod tests { ephemeral: false, has_rowid: true, unique: false, + index_method: None, }); available_indexes.insert("t1".to_string(), VecDeque::from([index])); diff --git a/core/translate/optimizer/mod.rs b/core/translate/optimizer/mod.rs index 798cd50f3..c0261a8dd 100644 --- a/core/translate/optimizer/mod.rs +++ b/core/translate/optimizer/mod.rs @@ -1076,6 +1076,7 @@ fn ephemeral_index_build( .table .btree() .is_some_and(|btree| btree.has_rowid), + index_method: None, }; ephemeral_index diff --git a/core/translate/order_by.rs b/core/translate/order_by.rs index 2dd7a3c8a..e2e8ccf15 100644 --- a/core/translate/order_by.rs +++ b/core/translate/order_by.rs @@ -109,6 +109,7 @@ pub fn init_order_by( unique: false, has_rowid: false, where_clause: None, + index_method: None, }); program.alloc_cursor_id(CursorType::BTreeIndex(index)) } else { diff --git a/core/translate/plan.rs b/core/translate/plan.rs index aa506ca76..623af3ec9 100644 --- a/core/translate/plan.rs +++ b/core/translate/plan.rs @@ -1000,6 +1000,9 @@ impl JoinedTable { if self.col_used_mask.is_empty() { return false; } + if index.index_method.is_some() { + return false; + } let mut index_cols_mask = ColumnUsedMask::default(); for col in index.columns.iter() { index_cols_mask.set(col.pos_in_table); diff --git a/core/util.rs b/core/util.rs index 426e5f8f6..8f0b5d9aa 100644 --- a/core/util.rs +++ b/core/util.rs @@ -182,7 +182,7 @@ pub fn parse_schema_rows( } } - schema.populate_indices(from_sql_indexes, automatic_indices)?; + schema.populate_indices(syms, from_sql_indexes, automatic_indices)?; schema.populate_materialized_views( materialized_view_info, dbsp_state_roots,