Merge 'Add support for pragma table-valued functions' from Piotr Rżysko

This PR adds support for table-valued functions for PRAGMAs (see the
[PRAGMA functions section](https://www.sqlite.org/pragma.html)).
Additionally, it introduces built-in table-valued functions. I
considered using extensions for this, but there are several reasons in
favor of a dedicated mechanism:
* It simplifies the use of internal functions, structs, etc. For
example, when implementing `json_each` and `json_tree`, direct access to
internals was necessary:
https://github.com/tursodatabase/limbo/pull/1088
* It avoids FFI overhead. [Benchmarks](https://github.com/piotrrzysko/li
mbo/blob/pragma_vtabs_bench/core/benches/pragma_benchmarks.rs) on my
hardware show that `pragma_table_info()` implemented as an extension is
2.5× slower than the built-in version.

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #1642

core/lib.rs

@@ -11,6 +11,7 @@ mod io;
 mod json;
 pub mod mvcc;
 mod parameters;
+mod pragma;
 mod pseudo;
 pub mod result;
 mod schema;
@@ -21,6 +22,7 @@ pub mod types;
 mod util;
 mod vdbe;
 mod vector;
+mod vtab;
 
 #[cfg(feature = "fuzz")]
 pub mod numeric;
@@ -32,6 +34,7 @@ mod numeric;
 #[global_allocator]
 static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
+use crate::vtab::VirtualTable;
 use crate::{fast_lock::SpinLock, translate::optimizer::optimize_plan};
 use core::str;
 pub use error::LimboError;
@@ -44,12 +47,9 @@ pub use io::UringIO;
 pub use io::{
     Buffer, Completion, File, MemoryIO, OpenFlags, PlatformIO, SyscallIO, WriteCompletion, IO,
 };
-use limbo_ext::{ConstraintInfo, IndexInfo, OrderByInfo, ResultCode, VTabKind, VTabModuleImpl};
 use limbo_sqlite3_parser::{ast, ast::Cmd, lexer::sql::Parser};
 use parking_lot::RwLock;
-use schema::{Column, Schema};
-use std::ffi::c_void;
-use std::rc::Weak;
+use schema::Schema;
 use std::{
     borrow::Cow,
     cell::{Cell, RefCell, UnsafeCell},
@@ -81,9 +81,10 @@ use tracing::{instrument, Level};
 use translate::select::prepare_select_plan;
 pub use types::RefValue;
 pub use types::Value;
-use util::{columns_from_create_table_body, parse_schema_rows};
+use util::parse_schema_rows;
+use vdbe::builder::QueryMode;
 use vdbe::builder::TableRefIdCounter;
-use vdbe::{builder::QueryMode, VTabOpaqueCursor};
+
 pub type Result<T, E = LimboError> = std::result::Result<T, E>;
 pub static DATABASE_VERSION: OnceLock<String> = OnceLock::new();
 
@@ -815,198 +816,6 @@ pub type Row = vdbe::Row;
 
 pub type StepResult = vdbe::StepResult;
 
-#[derive(Clone, Debug)]
-pub struct VirtualTable {
-    name: String,
-    args: Option<Vec<ast::Expr>>,
-    pub implementation: Rc<VTabModuleImpl>,
-    columns: Vec<Column>,
-    kind: VTabKind,
-    table_ptr: *const c_void,
-    connection_ptr: RefCell<Option<*mut limbo_ext::Conn>>,
-}
-
-impl Drop for VirtualTable {
-    fn drop(&mut self) {
-        if let Some(conn) = self.connection_ptr.borrow_mut().take() {
-            if conn.is_null() {
-                return;
-            }
-            // free the memory for the limbo_ext::Conn itself
-            let mut conn = unsafe { Box::from_raw(conn) };
-            // frees the boxed Weak pointer
-            conn.close();
-        }
-        *self.connection_ptr.borrow_mut() = None;
-    }
-}
-
-impl VirtualTable {
-    pub(crate) fn rowid(&self, cursor: &VTabOpaqueCursor) -> i64 {
-        unsafe { (self.implementation.rowid)(cursor.as_ptr()) }
-    }
-
-    pub(crate) fn best_index(
-        &self,
-        constraints: &[ConstraintInfo],
-        order_by: &[OrderByInfo],
-    ) -> IndexInfo {
-        unsafe {
-            IndexInfo::from_ffi((self.implementation.best_idx)(
-                constraints.as_ptr(),
-                constraints.len() as i32,
-                order_by.as_ptr(),
-                order_by.len() as i32,
-            ))
-        }
-    }
-
-    /// takes ownership of the provided Args
-    pub(crate) fn from_args(
-        tbl_name: Option<&str>,
-        module_name: &str,
-        args: Vec<limbo_ext::Value>,
-        syms: &SymbolTable,
-        kind: VTabKind,
-        exprs: Option<Vec<ast::Expr>>,
-    ) -> Result<Rc<Self>> {
-        let module = syms
-            .vtab_modules
-            .get(module_name)
-            .ok_or(LimboError::ExtensionError(format!(
-                "Virtual table module not found: {}",
-                module_name
-            )))?;
-        if let VTabKind::VirtualTable = kind {
-            if module.module_kind == VTabKind::TableValuedFunction {
-                return Err(LimboError::ExtensionError(format!(
-                    "{} is not a virtual table module",
-                    module_name
-                )));
-            }
-        };
-        let (schema, table_ptr) = module.implementation.create(args)?;
-        let mut parser = Parser::new(schema.as_bytes());
-        if let ast::Cmd::Stmt(ast::Stmt::CreateTable { body, .. }) = parser.next()?.ok_or(
-            LimboError::ParseError("Failed to parse schema from virtual table module".to_string()),
-        )? {
-            let columns = columns_from_create_table_body(&body)?;
-            let vtab = Rc::new(VirtualTable {
-                name: tbl_name.unwrap_or(module_name).to_owned(),
-                connection_ptr: RefCell::new(None),
-                implementation: module.implementation.clone(),
-                columns,
-                args: exprs,
-                kind,
-                table_ptr,
-            });
-            return Ok(vtab);
-        }
-        Err(crate::LimboError::ParseError(
-            "Failed to parse schema from virtual table module".to_string(),
-        ))
-    }
-
-    /// Accepts a Weak pointer to the connection that owns the VTable, that the module
-    /// can optionally use to query the other tables.
-    pub fn open(&self, conn: Weak<Connection>) -> crate::Result<VTabOpaqueCursor> {
-        // we need a Weak<Connection> to upgrade and call from the extension.
-        let weak_box: *mut Weak<Connection> = Box::into_raw(Box::new(conn));
-        let conn = limbo_ext::Conn::new(
-            weak_box.cast(),
-            crate::ext::prepare_stmt,
-            crate::ext::execute,
-            crate::ext::close,
-        );
-        let ext_conn_ptr = Box::into_raw(Box::new(conn));
-        // store the leaked connection pointer on the table so it can be freed on drop
-        *self.connection_ptr.borrow_mut() = Some(ext_conn_ptr);
-        let cursor = unsafe { (self.implementation.open)(self.table_ptr, ext_conn_ptr) };
-        VTabOpaqueCursor::new(cursor, self.implementation.close)
-    }
-
-    #[tracing::instrument(skip(cursor))]
-    pub fn filter(
-        &self,
-        cursor: &VTabOpaqueCursor,
-        idx_num: i32,
-        idx_str: Option<String>,
-        arg_count: usize,
-        args: Vec<limbo_ext::Value>,
-    ) -> Result<bool> {
-        tracing::trace!("xFilter");
-        let c_idx_str = idx_str
-            .map(|s| std::ffi::CString::new(s).unwrap())
-            .map(|cstr| cstr.into_raw())
-            .unwrap_or(std::ptr::null_mut());
-        let rc = unsafe {
-            (self.implementation.filter)(
-                cursor.as_ptr(),
-                arg_count as i32,
-                args.as_ptr(),
-                c_idx_str,
-                idx_num,
-            )
-        };
-        for arg in args {
-            unsafe {
-                arg.__free_internal_type();
-            }
-        }
-        match rc {
-            ResultCode::OK => Ok(true),
-            ResultCode::EOF => Ok(false),
-            _ => Err(LimboError::ExtensionError(rc.to_string())),
-        }
-    }
-
-    pub fn column(&self, cursor: &VTabOpaqueCursor, column: usize) -> Result<Value> {
-        let val = unsafe { (self.implementation.column)(cursor.as_ptr(), column as u32) };
-        Value::from_ffi(val)
-    }
-
-    pub fn next(&self, cursor: &VTabOpaqueCursor) -> Result<bool> {
-        let rc = unsafe { (self.implementation.next)(cursor.as_ptr()) };
-        match rc {
-            ResultCode::OK => Ok(true),
-            ResultCode::EOF => Ok(false),
-            _ => Err(LimboError::ExtensionError("Next failed".to_string())),
-        }
-    }
-
-    pub fn update(&self, args: &[Value]) -> Result<Option<i64>> {
-        let arg_count = args.len();
-        let ext_args = args.iter().map(|arg| arg.to_ffi()).collect::<Vec<_>>();
-        let newrowid = 0i64;
-        let rc = unsafe {
-            (self.implementation.update)(
-                self.table_ptr,
-                arg_count as i32,
-                ext_args.as_ptr(),
-                &newrowid as *const _ as *mut i64,
-            )
-        };
-        for arg in ext_args {
-            unsafe {
-                arg.__free_internal_type();
-            }
-        }
-        match rc {
-            ResultCode::OK => Ok(None),
-            ResultCode::RowID => Ok(Some(newrowid)),
-            _ => Err(LimboError::ExtensionError(rc.to_string())),
-        }
-    }
-
-    pub fn destroy(&self) -> Result<()> {
-        let rc = unsafe { (self.implementation.destroy)(self.table_ptr) };
-        match rc {
-            ResultCode::OK => Ok(()),
-            _ => Err(LimboError::ExtensionError(rc.to_string())),
-        }
-    }
-}
-
 pub struct SymbolTable {
     pub functions: HashMap<String, Rc<function::ExternalFunc>>,
     pub vtabs: HashMap<String, Rc<VirtualTable>>,

core/pragma.rs

@@ -0,0 +1,246 @@
+use crate::{Connection, LimboError, Statement, StepResult, Value};
+use bitflags::bitflags;
+use limbo_sqlite3_parser::ast::PragmaName;
+use std::rc::{Rc, Weak};
+use std::str::FromStr;
+
+bitflags! {
+    // Flag names match those used in SQLite:
+    // https://github.com/sqlite/sqlite/blob/b3c1884b65400da85636458298bd77cbbfdfb401/tool/mkpragmatab.tcl#L22-L29
+    struct PragmaFlags: u8 {
+        const NeedSchema = 0x01;
+        const NoColumns  = 0x02;
+        const NoColumns1 = 0x04;
+        const ReadOnly   = 0x08;
+        const Result0    = 0x10;
+        const Result1    = 0x20;
+        const SchemaOpt  = 0x40;
+        const SchemaReq  = 0x80;
+    }
+}
+
+struct Pragma {
+    flags: PragmaFlags,
+    columns: &'static [&'static str],
+}
+
+impl Pragma {
+    const fn new(flags: PragmaFlags, columns: &'static [&'static str]) -> Self {
+        Self { flags, columns }
+    }
+}
+
+fn pragma_for(pragma: PragmaName) -> Pragma {
+    use PragmaName::*;
+
+    match pragma {
+        CacheSize => Pragma::new(
+            PragmaFlags::NeedSchema
+                | PragmaFlags::Result0
+                | PragmaFlags::SchemaReq
+                | PragmaFlags::NoColumns1,
+            &["cache_size"],
+        ),
+        JournalMode => Pragma::new(
+            PragmaFlags::NeedSchema | PragmaFlags::Result0 | PragmaFlags::SchemaReq,
+            &["journal_mode"],
+        ),
+        LegacyFileFormat => {
+            unreachable!("pragma_for() called with LegacyFileFormat, which is unsupported")
+        }
+        PageCount => Pragma::new(
+            PragmaFlags::NeedSchema | PragmaFlags::Result0 | PragmaFlags::SchemaReq,
+            &["page_count"],
+        ),
+        PageSize => Pragma::new(
+            PragmaFlags::Result0 | PragmaFlags::SchemaReq | PragmaFlags::NoColumns1,
+            &["page_size"],
+        ),
+        SchemaVersion => Pragma::new(
+            PragmaFlags::NoColumns1 | PragmaFlags::Result0,
+            &["schema_version"],
+        ),
+        TableInfo => Pragma::new(
+            PragmaFlags::NeedSchema | PragmaFlags::Result1 | PragmaFlags::SchemaOpt,
+            &["cid", "name", "type", "notnull", "dflt_value", "pk"],
+        ),
+        UserVersion => Pragma::new(
+            PragmaFlags::NoColumns1 | PragmaFlags::Result0,
+            &["user_version"],
+        ),
+        WalCheckpoint => Pragma::new(PragmaFlags::NeedSchema, &["busy", "log", "checkpointed"]),
+    }
+}
+
+#[derive(Debug, Clone)]
+pub(crate) struct PragmaVirtualTable {
+    pragma_name: String,
+    visible_column_count: usize,
+    max_arg_count: usize,
+    has_pragma_arg: bool,
+}
+
+impl PragmaVirtualTable {
+    pub(crate) fn create(pragma_name: &str) -> crate::Result<(Self, String)> {
+        if let Ok(pragma) = PragmaName::from_str(pragma_name) {
+            if pragma == PragmaName::LegacyFileFormat {
+                return Err(Self::no_such_pragma(pragma_name));
+            }
+            let pragma = pragma_for(pragma);
+            if pragma
+                .flags
+                .intersects(PragmaFlags::Result0 | PragmaFlags::Result1)
+            {
+                let mut max_arg_count = 0;
+                let mut has_pragma_arg = false;
+
+                let mut sql = String::from("CREATE TABLE x(");
+                let col_defs = pragma
+                    .columns
+                    .iter()
+                    .map(|col| format!("\"{col}\""))
+                    .collect::<Vec<_>>()
+                    .join(", ");
+                sql.push_str(&col_defs);
+                if pragma.flags.contains(PragmaFlags::Result1) {
+                    sql.push_str(", arg HIDDEN");
+                    max_arg_count += 1;
+                    has_pragma_arg = true;
+                }
+                if pragma
+                    .flags
+                    .intersects(PragmaFlags::SchemaOpt | PragmaFlags::SchemaReq)
+                {
+                    sql.push_str(", schema HIDDEN");
+                    max_arg_count += 1;
+                }
+                sql.push(')');
+
+                return Ok((
+                    PragmaVirtualTable {
+                        pragma_name: pragma_name.to_owned(),
+                        visible_column_count: pragma.columns.len(),
+                        max_arg_count,
+                        has_pragma_arg,
+                    },
+                    sql,
+                ));
+            }
+        }
+        Err(Self::no_such_pragma(pragma_name))
+    }
+
+    fn no_such_pragma(pragma_name: &str) -> LimboError {
+        LimboError::ParseError(format!(
+            "No such table-valued function: pragma_{}",
+            pragma_name
+        ))
+    }
+
+    pub(crate) fn open(&self, conn: Weak<Connection>) -> crate::Result<PragmaVirtualTableCursor> {
+        Ok(PragmaVirtualTableCursor {
+            pragma_name: self.pragma_name.clone(),
+            pos: 0,
+            conn: conn
+                .upgrade()
+                .ok_or_else(|| LimboError::InternalError("Connection was dropped".into()))?,
+            stmt: None,
+            arg: None,
+            visible_column_count: self.visible_column_count,
+            max_arg_count: self.max_arg_count,
+            has_pragma_arg: self.has_pragma_arg,
+        })
+    }
+}
+
+pub struct PragmaVirtualTableCursor {
+    pragma_name: String,
+    pos: usize,
+    conn: Rc<Connection>,
+    stmt: Option<Statement>,
+    arg: Option<String>,
+    visible_column_count: usize,
+    max_arg_count: usize,
+    has_pragma_arg: bool,
+}
+
+impl PragmaVirtualTableCursor {
+    pub(crate) fn rowid(&self) -> i64 {
+        self.pos as i64
+    }
+
+    pub(crate) fn next(&mut self) -> crate::Result<bool> {
+        let stmt = self
+            .stmt
+            .as_mut()
+            .ok_or_else(|| LimboError::InternalError("Statement is missing".into()))?;
+        let result = stmt.step()?;
+        match result {
+            StepResult::Done => Ok(false),
+            _ => {
+                self.pos += 1;
+                Ok(true)
+            }
+        }
+    }
+
+    pub(crate) fn column(&self, idx: usize) -> crate::Result<Value> {
+        if idx < self.visible_column_count {
+            let value = self
+                .stmt
+                .as_ref()
+                .ok_or_else(|| LimboError::InternalError("Statement is missing".into()))?
+                .row()
+                .ok_or_else(|| LimboError::InternalError("No row available".into()))?
+                .get_value(idx)
+                .clone();
+            return Ok(value);
+        }
+
+        let value = match idx - self.visible_column_count {
+            0 => self
+                .arg
+                .as_ref()
+                .map_or(Value::Null, |arg| Value::from_text(arg)),
+            _ => Value::Null,
+        };
+        Ok(value)
+    }
+
+    pub(crate) fn filter(&mut self, args: Vec<Value>) -> crate::Result<bool> {
+        if args.len() > self.max_arg_count {
+            return Err(LimboError::ParseError(format!(
+                "Too many arguments for pragma {}: expected at most {}, got {}",
+                self.pragma_name,
+                self.max_arg_count,
+                args.len()
+            )));
+        }
+
+        let to_text = |v: &Value| v.to_text().map(str::to_owned);
+        let (arg, schema) = match args.as_slice() {
+            [arg0] if self.has_pragma_arg => (to_text(arg0), None),
+            [arg0] => (None, to_text(arg0)),
+            [arg0, arg1] => (to_text(arg0), to_text(arg1)),
+            _ => (None, None),
+        };
+
+        self.arg = arg;
+
+        if let Some(schema) = schema {
+            // Schema-qualified PRAGMA statements are not supported yet
+            return Err(LimboError::ParseError(format!(
+                "Schema argument is not supported yet (got schema: '{schema}')"
+            )));
+        }
+
+        let mut sql = format!("PRAGMA {}", self.pragma_name);
+        if let Some(arg) = &self.arg {
+            sql.push_str(&format!("=\"{}\"", arg));
+        }
+
+        self.stmt = Some(self.conn.prepare(sql)?);
+
+        self.next()
+    }
+}

core/translate/planner.rs

@@ -14,7 +14,7 @@ use crate::{
     function::Func,
     schema::{Schema, Table},
     translate::expr::walk_expr_mut,
-    util::{exprs_are_equivalent, normalize_ident, vtable_args},
+    util::{exprs_are_equivalent, normalize_ident},
     vdbe::{builder::TableRefIdCounter, BranchOffset},
     Result,
 };
@@ -344,18 +344,7 @@ fn parse_from_clause_table<'a>(
         }
         ast::SelectTable::TableCall(qualified_name, maybe_args, maybe_alias) => {
             let normalized_name = &normalize_ident(qualified_name.name.0.as_str());
-            let args = match maybe_args {
-                Some(ref args) => vtable_args(args),
-                None => vec![],
-            };
-            let vtab = crate::VirtualTable::from_args(
-                None,
-                normalized_name,
-                args,
-                syms,
-                limbo_ext::VTabKind::TableValuedFunction,
-                maybe_args,
-            )?;
+            let vtab = crate::VirtualTable::function(normalized_name, maybe_args, syms)?;
             let alias = maybe_alias
                 .as_ref()
                 .map(|a| match a {

core/types.rs

@@ -10,7 +10,8 @@ use crate::storage::sqlite3_ondisk::write_varint;
 use crate::translate::collate::CollationSeq;
 use crate::translate::plan::IterationDirection;
 use crate::vdbe::sorter::Sorter;
-use crate::vdbe::{Register, VTabOpaqueCursor};
+use crate::vdbe::Register;
+use crate::vtab::VirtualTableCursor;
 use crate::Result;
 use std::fmt::Display;
 
@@ -1394,7 +1395,7 @@ pub enum Cursor {
     BTree(BTreeCursor),
     Pseudo(PseudoCursor),
     Sorter(Sorter),
-    Virtual(VTabOpaqueCursor),
+    Virtual(VirtualTableCursor),
 }
 
 impl Cursor {
@@ -1431,7 +1432,7 @@ impl Cursor {
         }
     }
 
-    pub fn as_virtual_mut(&mut self) -> &mut VTabOpaqueCursor {
+    pub fn as_virtual_mut(&mut self) -> &mut VirtualTableCursor {
         match self {
             Self::Virtual(cursor) => cursor,
             _ => panic!("Cursor is not a virtual cursor"),

core/util.rs

@@ -82,27 +82,12 @@ pub fn parse_schema_rows(
                                     vtab.clone()
                                 } else {
                                     let mod_name = module_name_from_sql(sql)?;
-                                    if let Some(vmod) = syms.vtab_modules.get(mod_name) {
-                                        if let limbo_ext::VTabKind::VirtualTable = vmod.module_kind
-                                        {
-                                            crate::VirtualTable::from_args(
-                                                Some(name),
-                                                mod_name,
-                                                module_args_from_sql(sql)?,
-                                                syms,
-                                                vmod.module_kind,
-                                                None,
-                                            )?
-                                        } else {
-                                            return Err(LimboError::Corrupt("Table valued function: {name} registered as virtual table in schema".to_string()));
-                                        }
-                                    } else {
-                                        // the extension isn't loaded, so we emit a warning.
-                                        return Err(LimboError::ExtensionError(format!(
-                                            "Virtual table module '{}' not found\nPlease load extension",
-                                            &mod_name
-                                        )));
-                                    }
+                                    crate::VirtualTable::table(
+                                        Some(name),
+                                        mod_name,
+                                        module_args_from_sql(sql)?,
+                                        syms,
+                                    )?
                                 };
                                 schema.add_virtual_table(vtab);
                             } else {

core/vdbe/execute.rs

@@ -1040,23 +1040,8 @@ pub fn op_vcreate(
             "Failed to upgrade Connection".to_string(),
         ));
     };
-    let mod_type = conn
-        .syms
-        .borrow()
-        .vtab_modules
-        .get(&module_name)
-        .ok_or_else(|| {
-            crate::LimboError::ExtensionError(format!("Module {} not found", module_name))
-        })?
-        .module_kind;
-    let table = crate::VirtualTable::from_args(
-        Some(&table_name),
-        &module_name,
-        args,
-        &conn.syms.borrow(),
-        mod_type,
-        None,
-    )?;
+    let table =
+        crate::VirtualTable::table(Some(&table_name), &module_name, args, &conn.syms.borrow())?;
     {
         conn.syms
             .borrow_mut()
@@ -1085,28 +1070,19 @@ pub fn op_vfilter(
     else {
         unreachable!("unexpected Insn {:?}", insn)
     };
-    let (_, cursor_type) = program.cursor_ref.get(*cursor_id).unwrap();
-    let CursorType::VirtualTable(virtual_table) = cursor_type else {
-        panic!("VFilter on non-virtual table cursor");
-    };
     let has_rows = {
         let mut cursor = state.get_cursor(*cursor_id);
         let cursor = cursor.as_virtual_mut();
         let mut args = Vec::with_capacity(*arg_count);
         for i in 0..*arg_count {
-            args.push(
-                state.registers[args_reg + i]
-                    .get_owned_value()
-                    .clone()
-                    .to_ffi(),
-            );
+            args.push(state.registers[args_reg + i].get_owned_value().clone());
         }
         let idx_str = if let Some(idx_str) = idx_str {
             Some(state.registers[*idx_str].get_owned_value().to_string())
         } else {
             None
         };
-        virtual_table.filter(cursor, *idx_num as i32, idx_str, *arg_count, args)?
+        cursor.filter(*idx_num as i32, idx_str, *arg_count, args)?
     };
     if !has_rows {
         state.pc = pc_if_empty.to_offset_int();
@@ -1131,14 +1107,10 @@ pub fn op_vcolumn(
     else {
         unreachable!("unexpected Insn {:?}", insn)
     };
-    let (_, cursor_type) = program.cursor_ref.get(*cursor_id).unwrap();
-    let CursorType::VirtualTable(virtual_table) = cursor_type else {
-        panic!("VColumn on non-virtual table cursor");
-    };
     let value = {
         let mut cursor = state.get_cursor(*cursor_id);
         let cursor = cursor.as_virtual_mut();
-        virtual_table.column(cursor, *column)?
+        cursor.column(*column)?
     };
     state.registers[*dest] = Register::Value(value);
     state.pc += 1;
@@ -1223,14 +1195,10 @@ pub fn op_vnext(
     else {
         unreachable!("unexpected Insn {:?}", insn)
     };
-    let (_, cursor_type) = program.cursor_ref.get(*cursor_id).unwrap();
-    let CursorType::VirtualTable(virtual_table) = cursor_type else {
-        panic!("VNext on non-virtual table cursor");
-    };
     let has_more = {
         let mut cursor = state.get_cursor(*cursor_id);
         let cursor = cursor.as_virtual_mut();
-        virtual_table.next(cursor)?
+        cursor.next()?
     };
     if has_more {
         state.pc = pc_if_next.to_offset_int();
@@ -1958,11 +1926,7 @@ pub fn op_row_id(
             state.registers[*dest] = Register::Value(Value::Null);
         }
     } else if let Some(Cursor::Virtual(virtual_cursor)) = cursors.get_mut(*cursor_id).unwrap() {
-        let (_, cursor_type) = program.cursor_ref.get(*cursor_id).unwrap();
-        let CursorType::VirtualTable(virtual_table) = cursor_type else {
-            panic!("VUpdate on non-virtual table cursor");
-        };
-        let rowid = virtual_table.rowid(virtual_cursor);
+        let rowid = virtual_cursor.rowid();
         if rowid != 0 {
             state.registers[*dest] = Register::Value(Value::Integer(rowid));
         } else {

core/vdbe/mod.rs

@@ -53,7 +53,6 @@ use regex::Regex;
 use std::{
     cell::{Cell, RefCell},
     collections::HashMap,
-    ffi::c_void,
     num::NonZero,
     ops::Deref,
     rc::{Rc, Weak},
@@ -203,37 +202,6 @@ impl<const N: usize> Bitfield<N> {
     }
 }
 
-type VTabOpaqueCursorCloseFn = unsafe extern "C" fn(*const c_void) -> limbo_ext::ResultCode;
-
-pub struct VTabOpaqueCursor {
-    cursor: *const c_void,
-    close: VTabOpaqueCursorCloseFn,
-}
-
-impl VTabOpaqueCursor {
-    pub fn new(cursor: *const c_void, close: VTabOpaqueCursorCloseFn) -> Result<Self> {
-        if cursor.is_null() {
-            return Err(LimboError::InternalError(
-                "VTabOpaqueCursor: cursor is null".into(),
-            ));
-        }
-        Ok(Self { cursor, close })
-    }
-
-    pub fn as_ptr(&self) -> *const c_void {
-        self.cursor
-    }
-}
-
-impl Drop for VTabOpaqueCursor {
-    fn drop(&mut self) {
-        let result = unsafe { (self.close)(self.cursor) };
-        if !result.is_ok() {
-            tracing::error!("Failed to close virtual table cursor");
-        }
-    }
-}
-
 #[derive(Copy, Clone, PartialEq, Eq)]
 /// The commit state of the program.
 /// There are two states:

core/vtab.rs

@@ -0,0 +1,370 @@
+use crate::pragma::{PragmaVirtualTable, PragmaVirtualTableCursor};
+use crate::schema::Column;
+use crate::util::{columns_from_create_table_body, vtable_args};
+use crate::{Connection, LimboError, SymbolTable, Value};
+use fallible_iterator::FallibleIterator;
+use limbo_ext::{ConstraintInfo, IndexInfo, OrderByInfo, ResultCode, VTabKind, VTabModuleImpl};
+use limbo_sqlite3_parser::{ast, lexer::sql::Parser};
+use std::cell::RefCell;
+use std::ffi::c_void;
+use std::rc::{Rc, Weak};
+
+#[derive(Debug, Clone)]
+enum VirtualTableType {
+    Pragma(PragmaVirtualTable),
+    External(ExtVirtualTable),
+}
+
+#[derive(Clone, Debug)]
+pub struct VirtualTable {
+    pub(crate) name: String,
+    pub(crate) args: Option<Vec<ast::Expr>>,
+    pub(crate) columns: Vec<Column>,
+    pub(crate) kind: VTabKind,
+    vtab_type: VirtualTableType,
+}
+
+impl VirtualTable {
+    pub(crate) fn function(
+        name: &str,
+        args: Option<Vec<ast::Expr>>,
+        syms: &SymbolTable,
+    ) -> crate::Result<Rc<VirtualTable>> {
+        let module = syms.vtab_modules.get(name);
+        let (vtab_type, schema) = if let Some(_) = module {
+            let ext_args = match args {
+                Some(ref args) => vtable_args(args),
+                None => vec![],
+            };
+            ExtVirtualTable::create(name, module, ext_args, VTabKind::TableValuedFunction)
+                .map(|(vtab, columns)| (VirtualTableType::External(vtab), columns))?
+        } else if let Some(pragma_name) = name.strip_prefix("pragma_") {
+            PragmaVirtualTable::create(pragma_name)
+                .map(|(vtab, columns)| (VirtualTableType::Pragma(vtab), columns))?
+        } else {
+            return Err(LimboError::ParseError(format!(
+                "No such table-valued function: {}",
+                name
+            )));
+        };
+
+        let vtab = VirtualTable {
+            name: name.to_owned(),
+            args,
+            columns: Self::resolve_columns(schema)?,
+            kind: VTabKind::TableValuedFunction,
+            vtab_type,
+        };
+        Ok(Rc::new(vtab))
+    }
+
+    pub fn table(
+        tbl_name: Option<&str>,
+        module_name: &str,
+        args: Vec<limbo_ext::Value>,
+        syms: &SymbolTable,
+    ) -> crate::Result<Rc<VirtualTable>> {
+        let module = syms.vtab_modules.get(module_name);
+        let (table, schema) =
+            ExtVirtualTable::create(module_name, module, args, VTabKind::VirtualTable)?;
+        let vtab = VirtualTable {
+            name: tbl_name.unwrap_or(module_name).to_owned(),
+            args: None,
+            columns: Self::resolve_columns(schema)?,
+            kind: VTabKind::VirtualTable,
+            vtab_type: VirtualTableType::External(table),
+        };
+        Ok(Rc::new(vtab))
+    }
+
+    fn resolve_columns(schema: String) -> crate::Result<Vec<Column>> {
+        let mut parser = Parser::new(schema.as_bytes());
+        if let ast::Cmd::Stmt(ast::Stmt::CreateTable { body, .. }) = parser.next()?.ok_or(
+            LimboError::ParseError("Failed to parse schema from virtual table module".to_string()),
+        )? {
+            columns_from_create_table_body(&body)
+        } else {
+            Err(LimboError::ParseError(
+                "Failed to parse schema from virtual table module".to_string(),
+            ))
+        }
+    }
+
+    pub(crate) fn open(&self, conn: Weak<Connection>) -> crate::Result<VirtualTableCursor> {
+        match &self.vtab_type {
+            VirtualTableType::Pragma(table) => Ok(VirtualTableCursor::Pragma(table.open(conn)?)),
+            VirtualTableType::External(table) => {
+                Ok(VirtualTableCursor::External(table.open(conn)?))
+            }
+        }
+    }
+
+    pub(crate) fn update(&self, args: &[Value]) -> crate::Result<Option<i64>> {
+        match &self.vtab_type {
+            VirtualTableType::Pragma(_) => Err(LimboError::ReadOnly),
+            VirtualTableType::External(table) => table.update(args),
+        }
+    }
+
+    pub(crate) fn destroy(&self) -> crate::Result<()> {
+        match &self.vtab_type {
+            VirtualTableType::Pragma(_) => Ok(()),
+            VirtualTableType::External(table) => table.destroy(),
+        }
+    }
+
+    pub(crate) fn best_index(
+        &self,
+        constraints: &[ConstraintInfo],
+        order_by: &[OrderByInfo],
+    ) -> IndexInfo {
+        match &self.vtab_type {
+            VirtualTableType::Pragma(_) => {
+                // SQLite tries to estimate cost and row count for pragma_ TVFs,
+                // but since Limbo doesn't have cost-based planning yet, this
+                // estimation is not currently implemented.
+                Default::default()
+            }
+            VirtualTableType::External(table) => table.best_index(constraints, order_by),
+        }
+    }
+}
+
+pub enum VirtualTableCursor {
+    Pragma(PragmaVirtualTableCursor),
+    External(ExtVirtualTableCursor),
+}
+
+impl VirtualTableCursor {
+    pub(crate) fn next(&mut self) -> crate::Result<bool> {
+        match self {
+            VirtualTableCursor::Pragma(cursor) => cursor.next(),
+            VirtualTableCursor::External(cursor) => cursor.next(),
+        }
+    }
+
+    pub(crate) fn rowid(&self) -> i64 {
+        match self {
+            VirtualTableCursor::Pragma(cursor) => cursor.rowid(),
+            VirtualTableCursor::External(cursor) => cursor.rowid(),
+        }
+    }
+
+    pub(crate) fn column(&self, column: usize) -> crate::Result<Value> {
+        match self {
+            VirtualTableCursor::Pragma(cursor) => cursor.column(column),
+            VirtualTableCursor::External(cursor) => cursor.column(column),
+        }
+    }
+
+    pub(crate) fn filter(
+        &mut self,
+        idx_num: i32,
+        idx_str: Option<String>,
+        arg_count: usize,
+        args: Vec<Value>,
+    ) -> crate::Result<bool> {
+        match self {
+            VirtualTableCursor::Pragma(cursor) => cursor.filter(args),
+            VirtualTableCursor::External(cursor) => {
+                cursor.filter(idx_num, idx_str, arg_count, args)
+            }
+        }
+    }
+}
+
+#[derive(Clone, Debug)]
+struct ExtVirtualTable {
+    implementation: Rc<VTabModuleImpl>,
+    table_ptr: *const c_void,
+    connection_ptr: RefCell<Option<*mut limbo_ext::Conn>>,
+}
+
+impl Drop for ExtVirtualTable {
+    fn drop(&mut self) {
+        if let Some(conn) = self.connection_ptr.borrow_mut().take() {
+            if conn.is_null() {
+                return;
+            }
+            // free the memory for the limbo_ext::Conn itself
+            let mut conn = unsafe { Box::from_raw(conn) };
+            // frees the boxed Weak pointer
+            conn.close();
+        }
+        *self.connection_ptr.borrow_mut() = None;
+    }
+}
+
+impl ExtVirtualTable {
+    fn best_index(&self, constraints: &[ConstraintInfo], order_by: &[OrderByInfo]) -> IndexInfo {
+        unsafe {
+            IndexInfo::from_ffi((self.implementation.best_idx)(
+                constraints.as_ptr(),
+                constraints.len() as i32,
+                order_by.as_ptr(),
+                order_by.len() as i32,
+            ))
+        }
+    }
+
+    /// takes ownership of the provided Args
+    fn create(
+        module_name: &str,
+        module: Option<&Rc<crate::ext::VTabImpl>>,
+        args: Vec<limbo_ext::Value>,
+        kind: VTabKind,
+    ) -> crate::Result<(Self, String)> {
+        let module = module.ok_or(LimboError::ExtensionError(format!(
+            "Virtual table module not found: {}",
+            module_name
+        )))?;
+        if kind != module.module_kind {
+            let expected = match kind {
+                VTabKind::VirtualTable => "virtual table",
+                VTabKind::TableValuedFunction => "table-valued function",
+            };
+            return Err(LimboError::ExtensionError(format!(
+                "{} is not a {} module",
+                module_name, expected
+            )));
+        }
+        let (schema, table_ptr) = module.implementation.create(args)?;
+        let vtab = ExtVirtualTable {
+            connection_ptr: RefCell::new(None),
+            implementation: module.implementation.clone(),
+            table_ptr,
+        };
+        Ok((vtab, schema))
+    }
+
+    /// Accepts a Weak pointer to the connection that owns the VTable, that the module
+    /// can optionally use to query the other tables.
+    fn open(&self, conn: Weak<Connection>) -> crate::Result<ExtVirtualTableCursor> {
+        // we need a Weak<Connection> to upgrade and call from the extension.
+        let weak_box: *mut Weak<Connection> = Box::into_raw(Box::new(conn));
+        let conn = limbo_ext::Conn::new(
+            weak_box.cast(),
+            crate::ext::prepare_stmt,
+            crate::ext::execute,
+            crate::ext::close,
+        );
+        let ext_conn_ptr = Box::into_raw(Box::new(conn));
+        // store the leaked connection pointer on the table so it can be freed on drop
+        *self.connection_ptr.borrow_mut() = Some(ext_conn_ptr);
+        let cursor = unsafe { (self.implementation.open)(self.table_ptr, ext_conn_ptr) };
+        ExtVirtualTableCursor::new(cursor, self.implementation.clone())
+    }
+
+    fn update(&self, args: &[Value]) -> crate::Result<Option<i64>> {
+        let arg_count = args.len();
+        let ext_args = args.iter().map(|arg| arg.to_ffi()).collect::<Vec<_>>();
+        let newrowid = 0i64;
+        let rc = unsafe {
+            (self.implementation.update)(
+                self.table_ptr,
+                arg_count as i32,
+                ext_args.as_ptr(),
+                &newrowid as *const _ as *mut i64,
+            )
+        };
+        for arg in ext_args {
+            unsafe {
+                arg.__free_internal_type();
+            }
+        }
+        match rc {
+            ResultCode::OK => Ok(None),
+            ResultCode::RowID => Ok(Some(newrowid)),
+            _ => Err(LimboError::ExtensionError(rc.to_string())),
+        }
+    }
+
+    fn destroy(&self) -> crate::Result<()> {
+        let rc = unsafe { (self.implementation.destroy)(self.table_ptr) };
+        match rc {
+            ResultCode::OK => Ok(()),
+            _ => Err(LimboError::ExtensionError(rc.to_string())),
+        }
+    }
+}
+
+pub struct ExtVirtualTableCursor {
+    cursor: *const c_void,
+    implementation: Rc<VTabModuleImpl>,
+}
+
+impl ExtVirtualTableCursor {
+    fn new(cursor: *const c_void, implementation: Rc<VTabModuleImpl>) -> crate::Result<Self> {
+        if cursor.is_null() {
+            return Err(LimboError::InternalError(
+                "VirtualTableCursor: cursor is null".into(),
+            ));
+        }
+        Ok(Self {
+            cursor,
+            implementation,
+        })
+    }
+
+    fn rowid(&self) -> i64 {
+        unsafe { (self.implementation.rowid)(self.cursor) }
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn filter(
+        &self,
+        idx_num: i32,
+        idx_str: Option<String>,
+        arg_count: usize,
+        args: Vec<Value>,
+    ) -> crate::Result<bool> {
+        tracing::trace!("xFilter");
+        let ext_args = args.iter().map(|arg| arg.to_ffi()).collect::<Vec<_>>();
+        let c_idx_str = idx_str
+            .map(|s| std::ffi::CString::new(s).unwrap())
+            .map(|cstr| cstr.into_raw())
+            .unwrap_or(std::ptr::null_mut());
+        let rc = unsafe {
+            (self.implementation.filter)(
+                self.cursor,
+                arg_count as i32,
+                ext_args.as_ptr(),
+                c_idx_str,
+                idx_num,
+            )
+        };
+        for arg in ext_args {
+            unsafe {
+                arg.__free_internal_type();
+            }
+        }
+        match rc {
+            ResultCode::OK => Ok(true),
+            ResultCode::EOF => Ok(false),
+            _ => Err(LimboError::ExtensionError(rc.to_string())),
+        }
+    }
+
+    fn column(&self, column: usize) -> crate::Result<Value> {
+        let val = unsafe { (self.implementation.column)(self.cursor, column as u32) };
+        Value::from_ffi(val)
+    }
+
+    fn next(&self) -> crate::Result<bool> {
+        let rc = unsafe { (self.implementation.next)(self.cursor) };
+        match rc {
+            ResultCode::OK => Ok(true),
+            ResultCode::EOF => Ok(false),
+            _ => Err(LimboError::ExtensionError("Next failed".to_string())),
+        }
+    }
+}
+
+impl Drop for ExtVirtualTableCursor {
+    fn drop(&mut self) {
+        let result = unsafe { (self.implementation.close)(self.cursor) };
+        if !result.is_ok() {
+            tracing::error!("Failed to close virtual table cursor");
+        }
+    }
+}

testing/cli_tests/extensions.py

@@ -315,7 +315,7 @@ def test_series():
     ext_path = "./target/debug/liblimbo_series"
     limbo.run_test_fn(
         "SELECT * FROM generate_series(1, 10);",
-        lambda res: "Virtual table module not found: generate_series" in res,
+        lambda res: "No such table-valued function: generate_series" in res,
     )
     limbo.execute_dot(f".load {ext_path}")
     limbo.run_test_fn(

testing/pragma.test

@@ -7,10 +7,18 @@ do_execsql_test pragma-cache-size {
   PRAGMA cache_size
 } {-2000}
 
+do_execsql_test pragma-function-cache-size {
+  SELECT * FROM pragma_cache_size()
+} {-2000}
+
 do_execsql_test pragma-update-journal-mode-wal {
   PRAGMA journal_mode=WAL
 } {wal}
 
+do_execsql_test pragma-function-update-journal-mode {
+  SELECT * FROM pragma_journal_mode()
+} {wal}
+
 do_execsql_test pragma-table-info-equal-syntax {
   PRAGMA table_info=sqlite_schema
 } {0|type|TEXT|0||0
@@ -29,10 +37,23 @@ do_execsql_test pragma-table-info-call-syntax {
 4|sql|TEXT|0||0
 }
 
+do_execsql_test pragma-function-table-info {
+  SELECT * FROM pragma_table_info('sqlite_schema')
+} {0|type|TEXT|0||0
+1|name|TEXT|0||0
+2|tbl_name|TEXT|0||0
+3|rootpage|INT|0||0
+4|sql|TEXT|0||0
+}
+
 do_execsql_test pragma-table-info-invalid-table {
   PRAGMA table_info=pekka
 } {}
 
+do_execsql_test pragma-function-table-info-invalid-table {
+  SELECT * FROM pragma_table_info('pekka')
+} {}
+
 # temporarily skip this test case. The issue is detailed in #1407
 #do_execsql_test_on_specific_db ":memory:" pragma-page-count-empty {
 #  PRAGMA page_count
@@ -65,3 +86,49 @@ do_execsql_test_on_specific_db ":memory:" pragma-user-version-float-value {
   PRAGMA user_version = 10.9;
   PRAGMA user_version;
 } {10}
+
+do_execsql_test pragma-legacy-file-format {
+  PRAGMA legacy_file_format
+} {}
+
+do_execsql_test_error_content pragma-function-legacy-file-format {
+  SELECT * FROM pragma_legacy_file_format()
+} {"No such table"}
+
+do_execsql_test_error_content pragma-function-too-many-arguments {
+  SELECT * FROM pragma_table_info('sqlite_schema', 'main', 'arg3')
+} {"Too many arguments"}
+
+do_execsql_test_error_content pragma-function-update {
+  SELECT * FROM pragma_wal_checkpoint()
+} {"No such table"}
+
+do_execsql_test pragma-function-nontext-argument {
+  SELECT * FROM pragma_table_info('sqlite_schema', NULL);
+} {0|type|TEXT|0||0
+1|name|TEXT|0||0
+2|tbl_name|TEXT|0||0
+3|rootpage|INT|0||0
+4|sql|TEXT|0||0
+}
+
+do_execsql_test pragma-function-no-arguments {
+  SELECT * FROM pragma_table_info();
+} {}
+
+do_execsql_test_on_specific_db ":memory:" pragma-function-argument-with-space {
+  CREATE TABLE "foo bar"(c0);
+  SELECT * FROM pragma_table_info('foo bar')
+} {0|c0||0||0}
+
+# If the argument passed to the first function call were simply concatenated with the underlying PRAGMA statement,
+# we would end up with: PRAGMA table_info='sqlite_schema';CREATE TABLE foo(c0);SELECT 'bar'. Depending on how many
+# statements are executed at once, at least one of the following would run:
+# - PRAGMA table_info='sqlite_schema';
+# - CREATE TABLE foo(c0);
+# - SELECT 'bar';
+# No output means that none of them were executed.
+do_execsql_test pragma-function-sql-injection {
+  SELECT * FROM pragma_table_info('sqlite_schema'';CREATE TABLE foo(c0);SELECT ''bar');
+  SELECT * FROM pragma_table_info('foo');
+} {}