index_experimental flag to enable index usages

Currently indexes are the bulk of the problem with `UPDATE` and
`DELETE`, while we work on fixing those it makes sense to disable
indexing since they are not stable. We want to try to make everything
else stable before we continue with indexing.

.github/workflows/rust.yml

@@ -93,7 +93,10 @@ jobs:
       - name: Test
         run: make test
         timeout-minutes: 20
-
+      - uses: "./.github/shared/install_sqlite"
+      - name: Test with index enabled
+        run: SQLITE_EXEC="scripts/limbo-sqlite3-index-experimental" make test
+        timeout-minutes: 20
   test-sqlite:
     runs-on: blacksmith-4vcpu-ubuntu-2404
     steps:

Makefile

@@ -36,6 +36,7 @@ check-wasm-target:
 
 limbo:
 	cargo build
+	cargo build --features index_experimental --bin limbo_index_experimental
 .PHONY: limbo
 
 limbo-c:

cli/Cargo.toml

@@ -17,6 +17,11 @@ dist = true
 name = "limbo"
 path = "main.rs"
 
+[[bin]]
+name = "limbo_index_experimental"
+path = "main.rs"
+required-features = ["index_experimental"]
+
 
 [dependencies]
 anyhow.workspace = true
@@ -51,6 +56,7 @@ toml_edit = {version = "0.22.24", features = ["serde"]}
 [features]
 default = ["io_uring"]
 io_uring = ["limbo_core/io_uring"]
+index_experimental = ["limbo_core/index_experimental"]
 
 [build-dependencies]
 syntect = "5.2.0"

core/Cargo.toml

@@ -15,6 +15,7 @@ path = "lib.rs"
 
 [features]
 default = ["fs", "uuid", "time", "json", "static"]
+index_experimental = []
 fs = ["limbo_ext/vfs"]
 json = []
 uuid = ["limbo_uuid/static"]

core/schema.rs

@@ -76,6 +76,7 @@ impl Schema {
         }
     }
 
+    #[cfg(feature = "index_experimental")]
     pub fn add_index(&mut self, index: Arc<Index>) {
         let table_name = normalize_ident(&index.table_name);
         self.indexes

core/translate/optimizer/mod.rs

@@ -241,15 +241,19 @@ fn optimize_table_access(
         let table_idx = join_order_member.original_idx;
         let access_method = &access_methods_arena.borrow()[best_access_methods[i]];
         if access_method.is_scan() {
-            let is_leftmost_table = i == 0;
-            let uses_index = access_method.index.is_some();
-            let source_table_is_from_clause_subquery = matches!(
-                &joined_tables[table_idx].table,
-                Table::FromClauseSubquery(_)
-            );
+            #[cfg(feature = "index_experimental")]
+            let try_to_build_ephemeral_index = {
+                let is_leftmost_table = i == 0;
+                let uses_index = access_method.index.is_some();
+                let source_table_is_from_clause_subquery = matches!(
+                    &joined_tables[table_idx].table,
+                    Table::FromClauseSubquery(_)
+                );
+                !is_leftmost_table && !uses_index && !source_table_is_from_clause_subquery
+            };
+            #[cfg(not(feature = "index_experimental"))]
+            let try_to_build_ephemeral_index = false;
 
-            let try_to_build_ephemeral_index =
-                !is_leftmost_table && !uses_index && !source_table_is_from_clause_subquery;
             if !try_to_build_ephemeral_index {
                 joined_tables[table_idx].op = Operation::Scan {
                     iter_dir: access_method.iter_dir,

core/util.rs

@@ -136,23 +136,31 @@ pub fn parse_schema_rows(
                 StepResult::Busy => break,
             }
         }
-        for UnparsedFromSqlIndex {
-            table_name,
-            root_page,
-            sql,
-        } in from_sql_indexes
+        #[cfg(feature = "index_experimental")]
         {
-            let table = schema.get_btree_table(&table_name).unwrap();
-            let index = schema::Index::from_sql(&sql, root_page as usize, table.as_ref())?;
-            schema.add_index(Arc::new(index));
-        }
-        for (table_name, indices) in automatic_indices {
-            let table = schema.get_btree_table(&table_name).unwrap();
-            let ret_index =
-                schema::Index::automatic_from_primary_key_and_unique(table.as_ref(), indices)?;
-            for index in ret_index {
+            for UnparsedFromSqlIndex {
+                table_name,
+                root_page,
+                sql,
+            } in from_sql_indexes
+            {
+                let table = schema.get_btree_table(&table_name).unwrap();
+                let index = schema::Index::from_sql(&sql, root_page as usize, table.as_ref())?;
                 schema.add_index(Arc::new(index));
             }
+            #[cfg(feature = "index_experimental")]
+            {
+                for (table_name, indices) in automatic_indices {
+                    let table = schema.get_btree_table(&table_name).unwrap();
+                    let ret_index = schema::Index::automatic_from_primary_key_and_unique(
+                        table.as_ref(),
+                        indices,
+                    )?;
+                    for index in ret_index {
+                        schema.add_index(Arc::new(index));
+                    }
+                }
+            }
         }
     }
     Ok(())

scripts/limbo-sqlite3-index-experimental

@@ -0,0 +1,8 @@
+#!/bin/bash
+
+# if RUST_LOG is non-empty, enable tracing output
+if [ -n "$RUST_LOG" ]; then
+   target/debug/limbo_index_experimental -m list -t testing/test.log "$@"
+else
+   target/debug/limbo_index_experimental -m list "$@"
+fi

testing/join.test

@@ -228,11 +228,11 @@ do_execsql_test left-join-constant-condition-true-inner-join-constant-condition-
     select u.first_name, p.name, u2.first_name from users u left join products as p on 1 join users u2 on 0 limit 5;
 } {}
 
-do_execsql_test join-utilizing-both-seekrowid-and-secondary-index {
-  select u.first_name, p.name from users u join products p on u.id = p.id and u.age > 70;
-} {Matthew|boots
-Nicholas|shorts
-Jamie|hat}
+#do_execsql_test join-utilizing-both-seekrowid-and-secondary-index {
+#  select u.first_name, p.name from users u join products p on u.id = p.id and u.age > 70;
+#} {Matthew|boots
+#Nicholas|shorts
+#Jamie|hat}
 
 # important difference between regular SELECT * join and a SELECT * USING join is that the join keys are deduplicated
 # from the result in the USING case.
@@ -282,4 +282,4 @@ do_execsql_test left-join-backwards-iteration {
     where users.id < 13 order by users.id desc limit 3;
 } {12|Alan|
 11|Travis|accessories
-10|Daniel|coat}
+10|Daniel|coat}

testing/orderby.test

@@ -142,11 +142,11 @@ do_execsql_test case-insensitive-alias {
     select u.first_name as fF, count(1) > 0 as cC from users u where fF = 'Jamie' group by fF order by cC;
 } {Jamie|1}
 
-do_execsql_test age_idx_order_desc {
-    select first_name from users order by age desc limit 3;
-} {Robert
-Sydney
-Matthew}
+#do_execsql_test age_idx_order_desc {
+#    select first_name from users order by age desc limit 3;
+#} {Robert
+#Sydney
+#Matthew}
 
 do_execsql_test rowid_or_integer_pk_desc {
     select first_name from users order by id desc limit 3;
@@ -163,19 +163,19 @@ do_execsql_test orderby_desc_verify_rows {
     select count(1) from (select * from users order by age desc)
 } {10000}
 
-do_execsql_test orderby_desc_with_offset {
-    select first_name, age from users order by age desc limit 3 offset 666;
-} {Francis|94
-Matthew|94
-Theresa|94}
+#do_execsql_test orderby_desc_with_offset {
+#    select first_name, age from users order by age desc limit 3 offset 666;
+#} {Francis|94
+#Matthew|94
+#Theresa|94}
 
-do_execsql_test orderby_desc_with_filter {
-    select first_name, age from users where age <= 50 order by age desc limit 5;
-} {Gerald|50
-Nicole|50
-Tammy|50
-Marissa|50
-Daniel|50}
+#do_execsql_test orderby_desc_with_filter {
+#    select first_name, age from users where age <= 50 order by age desc limit 5;
+#} {Gerald|50
+#Nicole|50
+#Tammy|50
+#Marissa|50
+#Daniel|50}
 
 do_execsql_test orderby_asc_with_filter_range {
     select first_name, age from users where age <= 50 and age >= 49 order by age asc limit 5;
@@ -210,4 +210,4 @@ do_execsql_test orderby_desc_regression_verify_order {
     select id from users where id < 100 order by id desc limit 3;
 } {99
 98
-97}
+97}