Allow arbitrarily many columns in a table

Use roaring bitmaps because ColumnUsedMask is likely to be
sparsely populated.

Cargo.lock

@@ -4450,6 +4450,7 @@ dependencies = [
  "rand_chacha 0.9.0",
  "regex",
  "regex-syntax",
+ "roaring",
  "rstest",
  "rusqlite",
  "rustix 1.0.7",

core/Cargo.toml

@@ -83,6 +83,7 @@ turso_parser = { workspace = true }
 aegis = "0.9.0"
 twox-hash = "2.1.1"
 intrusive-collections = "0.9.7"
+roaring = "0.11.2"
 
 [build-dependencies]
 chrono = { workspace = true, default-features = false }

core/translate/plan.rs

@@ -757,33 +757,25 @@ impl TableReferences {
     }
 }
 
-#[derive(Clone, Debug, Default, PartialEq, Eq)]
+#[derive(Clone, Debug, Default, PartialEq)]
 #[repr(transparent)]
-pub struct ColumnUsedMask(u128);
+pub struct ColumnUsedMask(roaring::RoaringBitmap);
 
 impl ColumnUsedMask {
     pub fn set(&mut self, index: usize) {
-        assert!(
-            index < 128,
-            "ColumnUsedMask only supports up to 128 columns"
-        );
-        self.0 |= 1 << index;
+        self.0.insert(index as u32);
     }
 
     pub fn get(&self, index: usize) -> bool {
-        assert!(
-            index < 128,
-            "ColumnUsedMask only supports up to 128 columns"
-        );
-        self.0 & (1 << index) != 0
+        self.0.contains(index as u32)
     }
 
     pub fn contains_all_set_bits_of(&self, other: &Self) -> bool {
-        self.0 & other.0 == other.0
+        other.0.is_subset(&self.0)
     }
 
     pub fn is_empty(&self) -> bool {
-        self.0 == 0
+        self.0.is_empty()
     }
 }
 

tests/integration/query_processing/test_read_path.rs

@@ -1,5 +1,5 @@
 use crate::common::{limbo_exec_rows, TempDatabase};
-use turso_core::{StepResult, Value};
+use turso_core::{LimboError, StepResult, Value};
 
 #[test]
 fn test_statement_reset_bind() -> anyhow::Result<()> {
@@ -920,3 +920,73 @@ fn test_max_joined_tables_limit() {
     };
     assert!(result.contains("Only up to 63 tables can be joined"));
 }
+
+#[test]
+/// Test that we can create and select from a table with 1000 columns.
+fn test_many_columns() {
+    let mut create_sql = String::from("CREATE TABLE test (");
+    for i in 0..1000 {
+        if i > 0 {
+            create_sql.push_str(", ");
+        }
+        create_sql.push_str(&format!("col{} INTEGER", i));
+    }
+    create_sql.push(')');
+
+    let tmp_db = TempDatabase::new("test_many_columns", false);
+    let conn = tmp_db.connect_limbo();
+    conn.execute(&create_sql).unwrap();
+
+    // Insert a row with values 0-999
+    let mut insert_sql = String::from("INSERT INTO test VALUES (");
+    for i in 0..1000 {
+        if i > 0 {
+            insert_sql.push_str(", ");
+        }
+        insert_sql.push_str(&i.to_string());
+    }
+    insert_sql.push(')');
+    conn.execute(&insert_sql).unwrap();
+
+    // Select every 100th column
+    let mut select_sql = String::from("SELECT ");
+    let mut first = true;
+    for i in (0..1000).step_by(100) {
+        if !first {
+            select_sql.push_str(", ");
+        }
+        select_sql.push_str(&format!("col{}", i));
+        first = false;
+    }
+    select_sql.push_str(" FROM test");
+
+    let mut rows = Vec::new();
+    let mut stmt = conn.prepare(&select_sql).unwrap();
+    loop {
+        match stmt.step().unwrap() {
+            StepResult::Row => {
+                let row = stmt.row().unwrap();
+                rows.push(row.get_values().cloned().collect::<Vec<_>>());
+            }
+            StepResult::IO => stmt.run_once().unwrap(),
+            _ => break,
+        }
+    }
+
+    // Verify we got values 0,100,200,...,900
+    assert_eq!(
+        rows,
+        vec![vec![
+            turso_core::Value::Integer(0),
+            turso_core::Value::Integer(100),
+            turso_core::Value::Integer(200),
+            turso_core::Value::Integer(300),
+            turso_core::Value::Integer(400),
+            turso_core::Value::Integer(500),
+            turso_core::Value::Integer(600),
+            turso_core::Value::Integer(700),
+            turso_core::Value::Integer(800),
+            turso_core::Value::Integer(900),
+        ]]
+    );
+}