From cd36fc26fdd7cde97736d54154d871fa5a1dc1e4 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Thu, 12 Jun 2025 23:08:03 +0800 Subject: [PATCH 1/8] support intersect operation for compound select --- core/translate/compound_select.rs | 113 +++++++++++++++++++++++++++--- 1 file changed, 104 insertions(+), 9 deletions(-) diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 531248f4d..f9bfa018b 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -202,6 +202,52 @@ fn emit_compound_select( program.preassign_label_to_next_insn(label_jump_over_dedupe); } } + CompoundOperator::Intersect => { + let mut target_cursor_id = None; + match right_most.query_destination { + QueryDestination::EphemeralIndex { cursor_id, .. } => { + target_cursor_id = Some(cursor_id); + } + QueryDestination::ResultRows => {} + QueryDestination::CoroutineYield { .. } => {} + } + + let (left_cursor_id, left_index) = create_dedupe_index(program, &right_most); + plan.query_destination = QueryDestination::EphemeralIndex { + cursor_id: left_cursor_id, + index: left_index.clone(), + }; + let compound_select = Plan::CompoundSelect { + left, + right_most: plan, + limit, + offset, + order_by, + }; + emit_compound_select( + program, + compound_select, + schema, + syms, + None, + yield_reg, + reg_result_cols_start, + )?; + + let (right_cursor_id, right_index) = create_dedupe_index(program, &right_most); + right_most.query_destination = QueryDestination::EphemeralIndex { + cursor_id: right_cursor_id, + index: right_index, + }; + emit_query(program, &mut right_most, &mut right_most_ctx)?; + read_intersect_rows( + program, + left_cursor_id, + &left_index, + right_cursor_id, + target_cursor_id, + ); + } _ => { crate::bail_parse_error!("unimplemented compound select operator: {:?}", operator); } @@ -218,19 +264,15 @@ fn emit_compound_select( Ok(()) } -/// Creates an ephemeral index that will be used to deduplicate the results of any sub-selects -/// that appear before the last UNION operator. -fn create_union_dedupe_index( - program: &mut ProgramBuilder, - first_select_in_compound: &SelectPlan, -) -> (usize, Arc) { +// Creates an ephemeral index that will be used to deduplicate the results of any sub-selects +fn create_dedupe_index(program: &mut ProgramBuilder, select: &SelectPlan) -> (usize, Arc) { let dedupe_index = Arc::new(Index { - columns: first_select_in_compound + columns: select .result_columns .iter() .map(|c| IndexColumn { name: c - .name(&first_select_in_compound.table_references) + .name(&select.table_references) .map(|n| n.to_string()) .unwrap_or_default(), order: SortOrder::Asc, @@ -239,7 +281,7 @@ fn create_union_dedupe_index( collation: None, // FIXME: this should be inferred }) .collect(), - name: "union_dedupe".to_string(), + name: "compound_dedupe".to_string(), root_page: 0, ephemeral: true, table_name: String::new(), @@ -312,3 +354,56 @@ fn read_deduplicated_union_rows( cursor_id: dedupe_cursor_id, }); } + +// Emits the bytecode for Reading rows from the intersection of two cursors. +fn read_intersect_rows( + program: &mut ProgramBuilder, + left_cursor_id: usize, + index: &Index, + right_cursor_id: usize, + target_cursor_id: Option, +) { + let label_close = program.allocate_label(); + let label_loop_start = program.allocate_label(); + program.emit_insn(Insn::Rewind { + cursor_id: left_cursor_id, + pc_if_empty: label_close, + }); + + program.preassign_label_to_next_insn(label_loop_start); + let row_content_reg = program.alloc_register(); + // we need to emit opcode RowData here + + let label_next = program.allocate_label(); + program.emit_insn(Insn::NotFound { + cursor_id: right_cursor_id, + target_pc: label_next, + record_reg: row_content_reg, + num_regs: 0, + }); + let cols_start_reg = program.alloc_registers(index.columns.len()); + for i in 0..index.columns.len() { + program.emit_insn(Insn::Column { + cursor_id: left_cursor_id, + column: i, + dest: cols_start_reg + i, + }); + } + program.emit_insn(Insn::ResultRow { + start_reg: cols_start_reg, + count: index.columns.len(), + }); + program.preassign_label_to_next_insn(label_next); + program.emit_insn(Insn::Next { + cursor_id: left_cursor_id, + pc_if_next: label_loop_start, + }); + + program.preassign_label_to_next_insn(label_close); + program.emit_insn(Insn::Close { + cursor_id: right_cursor_id, + }); + program.emit_insn(Insn::Close { + cursor_id: left_cursor_id, + }); +} From d4789d0a05bb3f8746cdadc053424b8703b2b1f9 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Mon, 16 Jun 2025 22:31:06 +0800 Subject: [PATCH 2/8] add tests --- Makefile | 4 +++ core/translate/compound_select.rs | 38 ++++++++++++++++----- core/translate/select.rs | 5 +-- testing/select.test | 57 +++++++++++++++++++++++++++++++ 4 files changed, 93 insertions(+), 11 deletions(-) diff --git a/Makefile b/Makefile index 51dbdc052..3a93b4434 100644 --- a/Makefile +++ b/Makefile @@ -66,6 +66,10 @@ test-compat: SQLITE_EXEC=$(SQLITE_EXEC) ./testing/all.test .PHONY: test-compat +test-select: + SQLITE_EXEC=$(SQLITE_EXEC) ./testing/select.test +.PHONY: test-select + test-vector: SQLITE_EXEC=$(SQLITE_EXEC) ./testing/vector.test .PHONY: test-vector diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index f9bfa018b..244bd663e 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -361,7 +361,7 @@ fn read_intersect_rows( left_cursor_id: usize, index: &Index, right_cursor_id: usize, - target_cursor_id: Option, + target_cursor: Option, ) { let label_close = program.allocate_label(); let label_loop_start = program.allocate_label(); @@ -372,8 +372,10 @@ fn read_intersect_rows( program.preassign_label_to_next_insn(label_loop_start); let row_content_reg = program.alloc_register(); - // we need to emit opcode RowData here - + program.emit_insn(Insn::RowData { + cursor_id: left_cursor_id, + dest: row_content_reg, + }); let label_next = program.allocate_label(); program.emit_insn(Insn::NotFound { cursor_id: right_cursor_id, @@ -381,18 +383,36 @@ fn read_intersect_rows( record_reg: row_content_reg, num_regs: 0, }); - let cols_start_reg = program.alloc_registers(index.columns.len()); - for i in 0..index.columns.len() { + let column_count = index.columns.len(); + let cols_start_reg = program.alloc_registers(column_count); + for i in 0..column_count { program.emit_insn(Insn::Column { cursor_id: left_cursor_id, column: i, dest: cols_start_reg + i, + default: None, + }); + } + if let Some(target_cursor_id) = target_cursor { + program.emit_insn(Insn::MakeRecord { + start_reg: cols_start_reg, + count: column_count, + dest_reg: row_content_reg, + index_name: None, + }); + program.emit_insn(Insn::IdxInsert { + cursor_id: target_cursor_id, + record_reg: row_content_reg, + unpacked_start: Some(cols_start_reg), + unpacked_count: Some(column_count as u16), + flags: Default::default(), + }); + } else { + program.emit_insn(Insn::ResultRow { + start_reg: cols_start_reg, + count: column_count, }); } - program.emit_insn(Insn::ResultRow { - start_reg: cols_start_reg, - count: index.columns.len(), - }); program.preassign_label_to_next_insn(label_next); program.emit_insn(Insn::Next { cursor_id: left_cursor_id, diff --git a/core/translate/select.rs b/core/translate/select.rs index 0a43f20cc..c08e597b0 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -127,12 +127,13 @@ pub fn prepare_select_plan( let mut left = Vec::with_capacity(compounds.len()); for CompoundSelect { select, operator } in compounds { - // TODO: add support for EXCEPT and INTERSECT + // TODO: add support for INTERSECT if operator != ast::CompoundOperator::UnionAll && operator != ast::CompoundOperator::Union + && operator != ast::CompoundOperator::Intersect { crate::bail_parse_error!( - "only UNION ALL and UNION are supported for compound SELECTs" + "only UNION ALL, UNION and INTERSECT are supported for compound SELECTs" ); } left.push((last, operator)); diff --git a/testing/select.test b/testing/select.test index 636058e23..1d1c5afb4 100755 --- a/testing/select.test +++ b/testing/select.test @@ -361,3 +361,60 @@ if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-s y|y} } +select * from t UNION select * from u UNION select * from v UNION ALL select * from t; +} {x|x +y|y +x|x +y|y} + +do_execsql_test_on_specific_db {:memory:} select-intersect-1 { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + + select * from t INTERSECT select * from u; +} {x|x} + +do_execsql_test_on_specific_db {:memory:} select-intersect-2 { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('a','x'),('y','y'); + + select * from t INTERSECT select * from u INTERSECT select * from v INTERSECT select * from t; +} {y|y} + +do_execsql_test_on_specific_db {:memory:} select-intersect-union { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); + + select * from t INTERSECT select * from u UNION select * from v; +} {x|x +z|z} + +do_execsql_test_on_specific_db {:memory:} select-union-intersect { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); + + select * from t UNION select * from u INTERSECT select * from v; +} {x|x} + +#do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { +# CREATE TABLE t(x TEXT, y TEXT); +# CREATE TABLE u(x TEXT, y TEXT); +# INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); +# INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); + +# select * from t INTERSECT select * from u limit 2; +#} {x|x} \ No newline at end of file From 1fcc2ddd907d2a6ca468e2f856e8f6ceb204d2d1 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Mon, 16 Jun 2025 23:03:04 +0800 Subject: [PATCH 3/8] support limit --- core/translate/compound_select.rs | 8 ++++++++ testing/select.test | 34 +++++++++++++++++++------------ 2 files changed, 29 insertions(+), 13 deletions(-) diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 244bd663e..0bb934e12 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -246,6 +246,7 @@ fn emit_compound_select( &left_index, right_cursor_id, target_cursor_id, + limit_ctx, ); } _ => { @@ -362,6 +363,7 @@ fn read_intersect_rows( index: &Index, right_cursor_id: usize, target_cursor: Option, + limit_ctx: Option, ) { let label_close = program.allocate_label(); let label_loop_start = program.allocate_label(); @@ -413,6 +415,12 @@ fn read_intersect_rows( count: column_count, }); } + if let Some(limit_ctx) = limit_ctx { + program.emit_insn(Insn::DecrJumpZero { + reg: limit_ctx.reg_limit, + target_pc: label_close, + }); + } program.preassign_label_to_next_insn(label_next); program.emit_insn(Insn::Next { cursor_id: left_cursor_id, diff --git a/testing/select.test b/testing/select.test index 1d1c5afb4..a6f9c4492 100755 --- a/testing/select.test +++ b/testing/select.test @@ -361,12 +361,6 @@ if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-s y|y} } -select * from t UNION select * from u UNION select * from v UNION ALL select * from t; -} {x|x -y|y -x|x -y|y} - do_execsql_test_on_specific_db {:memory:} select-intersect-1 { CREATE TABLE t(x TEXT, y TEXT); CREATE TABLE u(x TEXT, y TEXT); @@ -410,11 +404,25 @@ do_execsql_test_on_specific_db {:memory:} select-union-intersect { select * from t UNION select * from u INTERSECT select * from v; } {x|x} -#do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { -# CREATE TABLE t(x TEXT, y TEXT); -# CREATE TABLE u(x TEXT, y TEXT); -# INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); -# INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); +do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); -# select * from t INTERSECT select * from u limit 2; -#} {x|x} \ No newline at end of file + select * from t INTERSECT select * from u limit 2; +} {x|x +y|y} + +do_execsql_test_on_specific_db {:memory:} select-intersect-union-with-limit { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('d','d'),('e','e'), ('z','z'); + INSERT INTO v VALUES('a','a'),('b','b'); + + select * from t INTERSECT select * from u UNION select * from v limit 3; +} {a|a +b|b +z|z} \ No newline at end of file From 41def8895f2b44c4ff946c748fdf99d5f7bb44c4 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Tue, 17 Jun 2025 16:51:13 +0800 Subject: [PATCH 4/8] make intersect in compound work with insert --- Makefile | 4 ---- core/translate/compound_select.rs | 25 +++++++++++++++++-------- testing/insert.test | 24 ++++++++++++++++++++++++ testing/select.test | 24 ++++++++++++++++++++++++ 4 files changed, 65 insertions(+), 12 deletions(-) diff --git a/Makefile b/Makefile index 3a93b4434..51dbdc052 100644 --- a/Makefile +++ b/Makefile @@ -66,10 +66,6 @@ test-compat: SQLITE_EXEC=$(SQLITE_EXEC) ./testing/all.test .PHONY: test-compat -test-select: - SQLITE_EXEC=$(SQLITE_EXEC) ./testing/select.test -.PHONY: test-select - test-vector: SQLITE_EXEC=$(SQLITE_EXEC) ./testing/vector.test .PHONY: test-vector diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 0bb934e12..401b7a65f 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -158,7 +158,7 @@ fn emit_compound_select( crate::bail_parse_error!("UNION not supported without indexes"); } else { new_dedupe_index = true; - create_union_dedupe_index(program, &right_most) + create_dedupe_index(program, &right_most) } } }; @@ -204,12 +204,10 @@ fn emit_compound_select( } CompoundOperator::Intersect => { let mut target_cursor_id = None; - match right_most.query_destination { - QueryDestination::EphemeralIndex { cursor_id, .. } => { - target_cursor_id = Some(cursor_id); - } - QueryDestination::ResultRows => {} - QueryDestination::CoroutineYield { .. } => {} + if let QueryDestination::EphemeralIndex { cursor_id, .. } = + right_most.query_destination + { + target_cursor_id = Some(cursor_id); } let (left_cursor_id, left_index) = create_dedupe_index(program, &right_most); @@ -247,6 +245,7 @@ fn emit_compound_select( right_cursor_id, target_cursor_id, limit_ctx, + yield_reg, ); } _ => { @@ -364,6 +363,7 @@ fn read_intersect_rows( right_cursor_id: usize, target_cursor: Option, limit_ctx: Option, + yield_reg: Option, ) { let label_close = program.allocate_label(); let label_loop_start = program.allocate_label(); @@ -386,7 +386,11 @@ fn read_intersect_rows( num_regs: 0, }); let column_count = index.columns.len(); - let cols_start_reg = program.alloc_registers(column_count); + let cols_start_reg = if let Some(yield_reg) = yield_reg { + yield_reg + 1 + } else { + program.alloc_registers(column_count) + }; for i in 0..column_count { program.emit_insn(Insn::Column { cursor_id: left_cursor_id, @@ -409,6 +413,11 @@ fn read_intersect_rows( unpacked_count: Some(column_count as u16), flags: Default::default(), }); + } else if let Some(yield_reg) = yield_reg { + program.emit_insn(Insn::Yield { + yield_reg, + end_offset: BranchOffset::Offset(0), + }) } else { program.emit_insn(Insn::ResultRow { start_reg: cols_start_reg, diff --git a/testing/insert.test b/testing/insert.test index 709ea655b..00e8c95fc 100755 --- a/testing/insert.test +++ b/testing/insert.test @@ -338,6 +338,30 @@ if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-s 2|200} } +do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect { + CREATE TABLE t(a, b); + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; + SELECT * FROM t; +} {2|200} + +do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect-2 { + CREATE TABLE t(a, b); + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE t3(a, b); + + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t3 VALUES (2, 200), (4, 400); + INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; + SELECT * FROM t; +} {2|200} + do_execsql_test_on_specific_db {:memory:} negative-primary-integer-key { CREATE TABLE t(a INTEGER PRIMARY KEY); insert into t values (-2),(13); diff --git a/testing/select.test b/testing/select.test index a6f9c4492..24410c3dd 100755 --- a/testing/select.test +++ b/testing/select.test @@ -404,6 +404,30 @@ do_execsql_test_on_specific_db {:memory:} select-union-intersect { select * from t UNION select * from u INTERSECT select * from v; } {x|x} +do_execsql_test_on_specific_db {:memory:} select-union-all-intersect { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); + + select * from t UNION ALL select * from u INTERSECT select * from v; +} {x|x} + +do_execsql_test_on_specific_db {:memory:} select-intersect-union-all { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); + + select * from t INTERSECT select * from u UNION ALL select * from v; +} {x|x +x|x +z|z} + do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { CREATE TABLE t(x TEXT, y TEXT); CREATE TABLE u(x TEXT, y TEXT); From c060905d00a02815349b23c05e5995a8c40a0171 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sat, 21 Jun 2025 23:18:02 +0800 Subject: [PATCH 5/8] add INTERSECT to compound_select_fuzz --- tests/integration/fuzz/mod.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/fuzz/mod.rs b/tests/integration/fuzz/mod.rs index 5c31c0a20..f6221265d 100644 --- a/tests/integration/fuzz/mod.rs +++ b/tests/integration/fuzz/mod.rs @@ -584,7 +584,7 @@ mod tests { )); } - const COMPOUND_OPERATORS: [&str; 2] = [" UNION ALL ", " UNION "]; + const COMPOUND_OPERATORS: [&str; 3] = [" UNION ALL ", " UNION ", " INTERSECT "]; let mut query = String::new(); for (i, select_statement) in select_statements.iter().enumerate() { From 2c4847210fbc11fc6d6c1bb9a337f3a7769472f1 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sat, 21 Jun 2025 23:59:34 +0800 Subject: [PATCH 6/8] ajust code to accommodate index_experimental feature --- core/translate/compound_select.rs | 24 +++-- testing/insert.test | 48 +++++----- testing/select.test | 150 +++++++++++++++--------------- 3 files changed, 113 insertions(+), 109 deletions(-) diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 401b7a65f..628edb31e 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -154,12 +154,8 @@ fn emit_compound_select( (cursor_id, index.clone()) } _ => { - if !schema.indexes_enabled() { - crate::bail_parse_error!("UNION not supported without indexes"); - } else { - new_dedupe_index = true; - create_dedupe_index(program, &right_most) - } + new_dedupe_index = true; + create_dedupe_index(program, &right_most, schema)? } }; plan.query_destination = QueryDestination::EphemeralIndex { @@ -210,7 +206,7 @@ fn emit_compound_select( target_cursor_id = Some(cursor_id); } - let (left_cursor_id, left_index) = create_dedupe_index(program, &right_most); + let (left_cursor_id, left_index) = create_dedupe_index(program, &right_most, schema)?; plan.query_destination = QueryDestination::EphemeralIndex { cursor_id: left_cursor_id, index: left_index.clone(), @@ -232,7 +228,7 @@ fn emit_compound_select( reg_result_cols_start, )?; - let (right_cursor_id, right_index) = create_dedupe_index(program, &right_most); + let (right_cursor_id, right_index) = create_dedupe_index(program, &right_most, schema)?; right_most.query_destination = QueryDestination::EphemeralIndex { cursor_id: right_cursor_id, index: right_index, @@ -265,7 +261,15 @@ fn emit_compound_select( } // Creates an ephemeral index that will be used to deduplicate the results of any sub-selects -fn create_dedupe_index(program: &mut ProgramBuilder, select: &SelectPlan) -> (usize, Arc) { +fn create_dedupe_index( + program: &mut ProgramBuilder, + select: &SelectPlan, + schema: &Schema +) -> crate::Result<(usize, Arc)> { + if !schema.indexes_enabled { + crate::bail_parse_error!("UNION OR INTERSECT is not supported without indexes"); + } + let dedupe_index = Arc::new(Index { columns: select .result_columns @@ -293,7 +297,7 @@ fn create_dedupe_index(program: &mut ProgramBuilder, select: &SelectPlan) -> (us cursor_id, is_table: false, }); - (cursor_id, dedupe_index.clone()) + Ok((cursor_id, dedupe_index.clone())) } /// Emits the bytecode for reading deduplicated rows from the ephemeral index created for UNION operators. diff --git a/testing/insert.test b/testing/insert.test index 00e8c95fc..4f3fef7b1 100755 --- a/testing/insert.test +++ b/testing/insert.test @@ -336,32 +336,32 @@ if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-s SELECT * FROM t; } {1|100 2|200} + + do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect { + CREATE TABLE t(a, b); + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; + SELECT * FROM t; + } {2|200} + + do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect-2 { + CREATE TABLE t(a, b); + CREATE TABLE t1(a, b); + CREATE TABLE t2(a, b); + CREATE TABLE t3(a, b); + + INSERT INTO t1 VALUES (1, 100), (2, 200); + INSERT INTO t2 VALUES (2, 200), (3, 300); + INSERT INTO t3 VALUES (2, 200), (4, 400); + INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2 INTERSECT SELECT * FROM t3; + SELECT * FROM t; + } {2|200} } -do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect { - CREATE TABLE t(a, b); - CREATE TABLE t1(a, b); - CREATE TABLE t2(a, b); - - INSERT INTO t1 VALUES (1, 100), (2, 200); - INSERT INTO t2 VALUES (2, 200), (3, 300); - INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; - SELECT * FROM t; -} {2|200} - -do_execsql_test_on_specific_db {:memory:} insert_from_select_intersect-2 { - CREATE TABLE t(a, b); - CREATE TABLE t1(a, b); - CREATE TABLE t2(a, b); - CREATE TABLE t3(a, b); - - INSERT INTO t1 VALUES (1, 100), (2, 200); - INSERT INTO t2 VALUES (2, 200), (3, 300); - INSERT INTO t3 VALUES (2, 200), (4, 400); - INSERT INTO t SELECT * FROM t1 INTERSECT SELECT * FROM t2; - SELECT * FROM t; -} {2|200} - do_execsql_test_on_specific_db {:memory:} negative-primary-integer-key { CREATE TABLE t(a INTEGER PRIMARY KEY); insert into t values (-2),(13); diff --git a/testing/select.test b/testing/select.test index 24410c3dd..3ee64d068 100755 --- a/testing/select.test +++ b/testing/select.test @@ -359,94 +359,94 @@ if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-s y|y x|x y|y} -} -do_execsql_test_on_specific_db {:memory:} select-intersect-1 { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); + do_execsql_test_on_specific_db {:memory:} select-intersect-1 { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); - select * from t INTERSECT select * from u; -} {x|x} + select * from t INTERSECT select * from u; + } {x|x} -do_execsql_test_on_specific_db {:memory:} select-intersect-2 { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - CREATE TABLE v(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('y','y'); - INSERT INTO v VALUES('a','x'),('y','y'); + do_execsql_test_on_specific_db {:memory:} select-intersect-2 { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('y','y'); + INSERT INTO v VALUES('a','x'),('y','y'); - select * from t INTERSECT select * from u INTERSECT select * from v INTERSECT select * from t; -} {y|y} + select * from t INTERSECT select * from u INTERSECT select * from v INTERSECT select * from t; + } {y|y} -do_execsql_test_on_specific_db {:memory:} select-intersect-union { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - CREATE TABLE v(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-intersect-union { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t INTERSECT select * from u UNION select * from v; -} {x|x -z|z} + select * from t INTERSECT select * from u UNION select * from v; + } {x|x + z|z} -do_execsql_test_on_specific_db {:memory:} select-union-intersect { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - CREATE TABLE v(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-union-intersect { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t UNION select * from u INTERSECT select * from v; -} {x|x} + select * from t UNION select * from u INTERSECT select * from v; + } {x|x} -do_execsql_test_on_specific_db {:memory:} select-union-all-intersect { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - CREATE TABLE v(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-union-all-intersect { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t UNION ALL select * from u INTERSECT select * from v; -} {x|x} + select * from t UNION ALL select * from u INTERSECT select * from v; + } {x|x} -do_execsql_test_on_specific_db {:memory:} select-intersect-union-all { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - CREATE TABLE v(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'); - INSERT INTO u VALUES('x','x'),('z','y'); - INSERT INTO v VALUES('x','x'),('z','z'); + do_execsql_test_on_specific_db {:memory:} select-intersect-union-all { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'); + INSERT INTO u VALUES('x','x'),('z','y'); + INSERT INTO v VALUES('x','x'),('z','z'); - select * from t INTERSECT select * from u UNION ALL select * from v; -} {x|x -x|x -z|z} + select * from t INTERSECT select * from u UNION ALL select * from v; + } {x|x + x|x + z|z} -do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); - INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); + do_execsql_test_on_specific_db {:memory:} select-intersect-with-limit { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('x','x'),('y','y'), ('z','z'); - select * from t INTERSECT select * from u limit 2; -} {x|x -y|y} + select * from t INTERSECT select * from u limit 2; + } {x|x + y|y} -do_execsql_test_on_specific_db {:memory:} select-intersect-union-with-limit { - CREATE TABLE t(x TEXT, y TEXT); - CREATE TABLE u(x TEXT, y TEXT); - CREATE TABLE v(x TEXT, y TEXT); - INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); - INSERT INTO u VALUES('d','d'),('e','e'), ('z','z'); - INSERT INTO v VALUES('a','a'),('b','b'); + do_execsql_test_on_specific_db {:memory:} select-intersect-union-with-limit { + CREATE TABLE t(x TEXT, y TEXT); + CREATE TABLE u(x TEXT, y TEXT); + CREATE TABLE v(x TEXT, y TEXT); + INSERT INTO t VALUES('x','x'),('y','y'), ('z','z'); + INSERT INTO u VALUES('d','d'),('e','e'), ('z','z'); + INSERT INTO v VALUES('a','a'),('b','b'); - select * from t INTERSECT select * from u UNION select * from v limit 3; -} {a|a -b|b -z|z} \ No newline at end of file + select * from t INTERSECT select * from u UNION select * from v limit 3; + } {a|a + b|b + z|z} +} \ No newline at end of file From 51764d882e4b20e7a3967d72f6827f5112845277 Mon Sep 17 00:00:00 2001 From: meteorgan Date: Sun, 22 Jun 2025 17:52:13 +0800 Subject: [PATCH 7/8] fix comments --- core/translate/select.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/translate/select.rs b/core/translate/select.rs index c08e597b0..ad02010ca 100644 --- a/core/translate/select.rs +++ b/core/translate/select.rs @@ -127,7 +127,7 @@ pub fn prepare_select_plan( let mut left = Vec::with_capacity(compounds.len()); for CompoundSelect { select, operator } in compounds { - // TODO: add support for INTERSECT + // TODO: add support for EXCEPT if operator != ast::CompoundOperator::UnionAll && operator != ast::CompoundOperator::Union && operator != ast::CompoundOperator::Intersect From 0ed94f13f51a05a11545f617928d1745d7ef115a Mon Sep 17 00:00:00 2001 From: meteorgan Date: Thu, 26 Jun 2025 17:23:01 +0800 Subject: [PATCH 8/8] resolve conflicts --- core/translate/compound_select.rs | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/core/translate/compound_select.rs b/core/translate/compound_select.rs index 628edb31e..db84e1236 100644 --- a/core/translate/compound_select.rs +++ b/core/translate/compound_select.rs @@ -206,7 +206,8 @@ fn emit_compound_select( target_cursor_id = Some(cursor_id); } - let (left_cursor_id, left_index) = create_dedupe_index(program, &right_most, schema)?; + let (left_cursor_id, left_index) = + create_dedupe_index(program, &right_most, schema)?; plan.query_destination = QueryDestination::EphemeralIndex { cursor_id: left_cursor_id, index: left_index.clone(), @@ -228,7 +229,8 @@ fn emit_compound_select( reg_result_cols_start, )?; - let (right_cursor_id, right_index) = create_dedupe_index(program, &right_most, schema)?; + let (right_cursor_id, right_index) = + create_dedupe_index(program, &right_most, schema)?; right_most.query_destination = QueryDestination::EphemeralIndex { cursor_id: right_cursor_id, index: right_index, @@ -264,7 +266,7 @@ fn emit_compound_select( fn create_dedupe_index( program: &mut ProgramBuilder, select: &SelectPlan, - schema: &Schema + schema: &Schema, ) -> crate::Result<(usize, Arc)> { if !schema.indexes_enabled { crate::bail_parse_error!("UNION OR INTERSECT is not supported without indexes");