feat: impl python binding

refactor: pep-0249

refactor: rust comment and requirements-dev.txt

fix: name conflict

bindings/python/Cargo.toml

@@ -0,0 +1,25 @@
+[package]
+name = "py-limbo"
+version.workspace = true
+authors.workspace = true
+edition.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[lib]
+name = "_limbo"
+crate-type = ["cdylib"]
+
+[features]
+# must be enabled when building with `cargo build`, maturin enables this automatically
+extension-module = ["pyo3/extension-module"]
+
+[dependencies]
+anyhow = "1.0"
+limbo_core = { path = "../../core" }
+pyo3 = { version = "0.22.2", features = ["anyhow", "auto-initialize"] }
+
+[build-dependencies]
+version_check = "0.9.5"
+# used where logic has to be version/distribution specific, e.g. pypy
+pyo3-build-config = { version = "0.22.0" }

bindings/python/build.rs

@@ -0,0 +1,4 @@
+fn main() {
+    pyo3_build_config::use_pyo3_cfgs();
+    println!("cargo::rustc-check-cfg=cfg(allocator, values(\"default\", \"mimalloc\"))");
+}

bindings/python/limbo/__init__.py

@@ -0,0 +1,29 @@
+from _limbo import (
+    Connection,
+    Cursor,
+    DatabaseError,
+    DataError,
+    IntegrityError,
+    InterfaceError,
+    InternalError,
+    NotSupportedError,
+    OperationalError,
+    ProgrammingError,
+    __version__,
+    connect,
+)
+
+__all__ = [
+    "__version__",
+    "Connection",
+    "Cursor",
+    "InterfaceError",
+    "DatabaseError",
+    "DataError",
+    "OperationalError",
+    "IntegrityError",
+    "InternalError",
+    "ProgrammingError",
+    "NotSupportedError",
+    "connect",
+]

bindings/python/limbo/_limbo.pyi

@@ -0,0 +1,176 @@
+from typing import Any, List, Optional, Tuple
+
+__version__: str
+
+class Connection:
+    def cursor(self) -> "Cursor":
+        """
+        Creates a new cursor object using this connection.
+
+        :return: A new Cursor object.
+        :raises InterfaceError: If the cursor cannot be created.
+        """
+        ...
+
+    def close(self) -> None:
+        """
+        Closes the connection to the database.
+
+        :raises OperationalError: If there is an error closing the connection.
+        """
+        ...
+
+    def commit(self) -> None:
+        """
+        Commits the current transaction.
+
+        :raises OperationalError: If there is an error during commit.
+        """
+        ...
+
+    def rollback(self) -> None:
+        """
+        Rolls back the current transaction.
+
+        :raises OperationalError: If there is an error during rollback.
+        """
+        ...
+
+class Cursor:
+    arraysize: int
+    description: Optional[
+        Tuple[
+            str,
+            str,
+            Optional[str],
+            Optional[str],
+            Optional[str],
+            Optional[str],
+            Optional[str],
+        ]
+    ]
+    rowcount: int
+
+    def execute(
+        self, sql: str, parameters: Optional[Tuple[Any, ...]] = None
+    ) -> "Cursor":
+        """
+        Prepares and executes a SQL statement using the connection.
+
+        :param sql: The SQL query to execute.
+        :param parameters: The parameters to substitute into the SQL query.
+        :raises ProgrammingError: If there is an error in the SQL query.
+        :raises OperationalError: If there is an error executing the query.
+        :return: The cursor object.
+        """
+        ...
+
+    def executemany(
+        self, sql: str, parameters: Optional[List[Tuple[Any, ...]]] = None
+    ) -> None:
+        """
+        Executes a SQL command against all parameter sequences or mappings found in the sequence `parameters`.
+
+        :param sql: The SQL command to execute.
+        :param parameters: A list of parameter sequences or mappings.
+        :raises ProgrammingError: If there is an error in the SQL query.
+        :raises OperationalError: If there is an error executing the query.
+        """
+        ...
+
+    def fetchone(self) -> Optional[Tuple[Any, ...]]:
+        """
+        Fetches the next row from the result set.
+
+        :return: A tuple representing the next row, or None if no more rows are available.
+        :raises OperationalError: If there is an error fetching the row.
+        """
+        ...
+
+    def fetchall(self) -> List[Tuple[Any, ...]]:
+        """
+        Fetches all remaining rows from the result set.
+
+        :return: A list of tuples, each representing a row in the result set.
+        :raises OperationalError: If there is an error fetching the rows.
+        """
+        ...
+
+    def fetchmany(self, size: Optional[int] = None) -> List[Tuple[Any, ...]]:
+        """
+        Fetches the next set of rows of a size specified by the `arraysize` property.
+
+        :param size: Optional integer to specify the number of rows to fetch.
+        :return: A list of tuples, each representing a row in the result set.
+        :raises OperationalError: If there is an error fetching the rows.
+        """
+        ...
+
+    def close(self) -> None:
+        """
+        Closes the cursor.
+
+        :raises OperationalError: If there is an error closing the cursor.
+        """
+        ...
+
+# Exception classes
+class Warning(Exception):
+    """Exception raised for important warnings like data truncations while inserting."""
+
+    ...
+
+class Error(Exception):
+    """Base class for all other error exceptions. Catch all database-related errors using this class."""
+
+    ...
+
+class InterfaceError(Error):
+    """Exception raised for errors related to the database interface rather than the database itself."""
+
+    ...
+
+class DatabaseError(Error):
+    """Exception raised for errors that are related to the database."""
+
+    ...
+
+class DataError(DatabaseError):
+    """Exception raised for errors due to problems with the processed data like division by zero, numeric value out of range, etc."""
+
+    ...
+
+class OperationalError(DatabaseError):
+    """Exception raised for errors related to the database’s operation, not necessarily under the programmer's control."""
+
+    ...
+
+class IntegrityError(DatabaseError):
+    """Exception raised when the relational integrity of the database is affected, e.g., a foreign key check fails."""
+
+    ...
+
+class InternalError(DatabaseError):
+    """Exception raised when the database encounters an internal error, e.g., cursor is not valid anymore, transaction out of sync."""
+
+    ...
+
+class ProgrammingError(DatabaseError):
+    """Exception raised for programming errors, e.g., table not found, syntax error in SQL, wrong number of parameters specified."""
+
+    ...
+
+class NotSupportedError(DatabaseError):
+    """Exception raised when a method or database API is used which is not supported by the database."""
+
+    ...
+
+def connect(path: str) -> Connection:
+    """
+    Connects to a database at the specified path.
+
+    :param path: The path to the database file.
+    :return: A Connection object to the database.
+    :raises InterfaceError: If the database cannot be connected.
+    """
+    ...

bindings/python/pyproject.toml

@@ -0,0 +1,85 @@
+[build-system]
+requires = ['maturin>=1,<2', 'typing_extensions']
+build-backend = 'maturin'
+
+[project]
+name = 'limbo'
+description = "Limbo is a work-in-progress, in-process OLTP database management system, compatible with SQLite."
+requires-python = '>=3.8'
+classifiers = [
+    'Development Status :: 3 - Alpha',
+    'Programming Language :: Python',
+    'Programming Language :: Python :: 3',
+    'Programming Language :: Python :: 3 :: Only',
+    'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
+    'Programming Language :: Python :: 3.12',
+    'Programming Language :: Python :: 3.13',
+    'Programming Language :: Rust',
+    'License :: OSI Approved :: MIT License',
+    'Operating System :: POSIX :: Linux',
+    'Operating System :: Microsoft :: Windows',
+    'Operating System :: MacOS',
+    'Topic :: Database',
+    'Topic :: Software Development :: Libraries',
+    'Topic :: Software Development :: Libraries :: Python Modules',
+    'Topic :: Database :: Database Engines/Servers',
+]
+dependencies = ['typing-extensions >=4.6.0,!=4.7.0']
+dynamic = [
+    'readme',
+    'version'
+]
+
+[project.optional-dependencies]
+dev = [
+    "maturin==1.7.0",
+    "black==24.4.2",
+    "isort==5.13.2",
+    "mypy==1.11.0",
+    "pytest==8.3.1",
+    "pytest-cov==5.0.0",
+    "ruff==0.5.4"
+]
+
+[project.urls]
+Homepage = "https://github.com/penberg/limbo"
+Source = "https://github.com/penberg/limbo"
+
+[tool.maturin]
+bindings = 'pyo3'
+
+[tool.ruff]
+line-length = 120
+
+[tool.ruff.lint]
+extend-select = ['Q', 'RUF100', 'C90', 'I']
+extend-ignore = [
+    'E721', # using type() instead of isinstance() - we use this in tests
+]
+flake8-quotes = { inline-quotes = 'single', multiline-quotes = 'double' }
+mccabe = { max-complexity = 13 }
+isort = { known-first-party = ['pydantic_core', 'tests'] }
+
+[tool.ruff.format]
+quote-style = 'single'
+
+[tool.pytest.ini_options]
+testpaths = 'tests'
+log_format = '%(name)s %(levelname)s: %(message)s'
+
+
+[tool.coverage.run]
+source = ['limbo']
+branch = true
+
+[tool.coverage.report]
+precision = 2
+exclude_lines = [
+    'pragma: no cover',
+    'raise NotImplementedError',
+    'if TYPE_CHECKING:',
+    '@overload',
+]

bindings/python/requirements-dev.txt

@@ -0,0 +1,46 @@
+#
+# This file is autogenerated by pip-compile with Python 3.11
+# by the following command:
+#
+#    pip-compile --extra=dev --output-file=requirements-dev.txt --strip-extras pyproject.toml
+#
+black==24.4.2
+    # via limbo (pyproject.toml)
+click==8.1.7
+    # via black
+coverage==7.6.1
+    # via pytest-cov
+iniconfig==2.0.0
+    # via pytest
+isort==5.13.2
+    # via limbo (pyproject.toml)
+maturin==1.7.0
+    # via limbo (pyproject.toml)
+mypy==1.11.0
+    # via limbo (pyproject.toml)
+mypy-extensions==1.0.0
+    # via
+    #   black
+    #   mypy
+packaging==24.1
+    # via
+    #   black
+    #   pytest
+pathspec==0.12.1
+    # via black
+platformdirs==4.2.2
+    # via black
+pluggy==1.5.0
+    # via pytest
+pytest==8.3.1
+    # via
+    #   limbo (pyproject.toml)
+    #   pytest-cov
+pytest-cov==5.0.0
+    # via limbo (pyproject.toml)
+ruff==0.5.4
+    # via limbo (pyproject.toml)
+typing-extensions==4.12.2
+    # via
+    #   limbo (pyproject.toml)
+    #   mypy

bindings/python/src/errors.rs

@@ -0,0 +1,35 @@
+use pyo3::create_exception;
+use pyo3::exceptions::PyException;
+
+create_exception!(
+    limbo,
+    Warning,
+    PyException,
+    "Exception raised for important warnings like data truncations while inserting."
+);
+create_exception!(limbo, Error, PyException, "Base class for all other error exceptions. Catch all database-related errors using this class.");
+
+create_exception!(
+    limbo,
+    InterfaceError,
+    Error,
+    "Raised for errors related to the database interface rather than the database itself."
+);
+create_exception!(
+    limbo,
+    DatabaseError,
+    Error,
+    "Raised for errors that are related to the database."
+);
+
+create_exception!(limbo, DataError, DatabaseError, "Raised for errors due to problems with the processed data like division by zero, numeric value out of range, etc.");
+create_exception!(limbo, OperationalError, DatabaseError, "Raised for errors related to the database’s operation, not necessarily under the programmer's control.");
+create_exception!(limbo, IntegrityError, DatabaseError, "Raised when the relational integrity of the database is affected, e.g., a foreign key check fails.");
+create_exception!(limbo, InternalError, DatabaseError, "Raised when the database encounters an internal error, e.g., cursor is not valid anymore, transaction out of sync.");
+create_exception!(limbo, ProgrammingError, DatabaseError, "Raised for programming errors, e.g., table not found, syntax error in SQL, wrong number of parameters specified.");
+create_exception!(
+    limbo,
+    NotSupportedError,
+    DatabaseError,
+    "Raised when a method or database API is used which is not supported by the database."
+);

bindings/python/src/lib.rs

@@ -0,0 +1,264 @@
+use anyhow::Result;
+use errors::*;
+use limbo_core::IO;
+use pyo3::prelude::*;
+use pyo3::types::PyList;
+use pyo3::types::PyTuple;
+use std::sync::{Arc, Mutex};
+
+mod errors;
+
+#[pyclass]
+#[derive(Clone, Debug)]
+struct Description {
+    #[pyo3(get)]
+    name: String,
+    #[pyo3(get)]
+    type_code: String,
+    #[pyo3(get)]
+    display_size: Option<String>,
+    #[pyo3(get)]
+    internal_size: Option<String>,
+    #[pyo3(get)]
+    precision: Option<String>,
+    #[pyo3(get)]
+    scale: Option<String>,
+    #[pyo3(get)]
+    null_ok: Option<String>,
+}
+
+impl IntoPy<Py<PyTuple>> for Description {
+    fn into_py(self, py: Python<'_>) -> Py<PyTuple> {
+        PyTuple::new_bound(
+            py,
+            vec![
+                self.name.into_py(py),
+                self.type_code.into_py(py),
+                self.display_size.into_py(py),
+                self.internal_size.into_py(py),
+                self.precision.into_py(py),
+                self.scale.into_py(py),
+                self.null_ok.into_py(py),
+            ],
+        )
+        .into()
+    }
+}
+
+#[pyclass]
+pub struct Cursor {
+    /// This read/write attribute specifies the number of rows to fetch at a time with `.fetchmany()`.
+    /// It defaults to `1`, meaning it fetches a single row at a time.
+    #[pyo3(get)]
+    arraysize: i64,
+
+    conn: Connection,
+
+    /// The `.description` attribute is a read-only sequence of 7-item, each describing a column in the result set:
+    ///
+    /// - `name`: The column's name (always present).
+    /// - `type_code`: The data type code (always present).
+    /// - `display_size`: Column's display size (optional).
+    /// - `internal_size`: Column's internal size (optional).
+    /// - `precision`: Numeric precision (optional).
+    /// - `scale`: Numeric scale (optional).
+    /// - `null_ok`: Indicates if null values are allowed (optional).
+    ///
+    /// The `name` and `type_code` fields are mandatory; others default to `None` if not applicable.
+    ///
+    /// This attribute is `None` for operations that do not return rows or if no `.execute*()` method has been invoked.
+    #[pyo3(get)]
+    description: Option<Description>,
+
+    /// Read-only attribute that provides the number of modified rows for `INSERT`, `UPDATE`, `DELETE`,
+    /// and `REPLACE` statements; it is `-1` for other statements, including CTE queries.
+    /// It is only updated by the `execute()` and `executemany()` methods after the statement has run to completion.
+    /// This means any resulting rows must be fetched for `rowcount` to be updated.
+    #[pyo3(get)]
+    rowcount: i64,
+
+    smt: Option<Arc<Mutex<limbo_core::Statement>>>,
+}
+
+// SAFETY: The limbo_core crate guarantees that `Cursor` is thread-safe.
+unsafe impl Send for Cursor {}
+
+#[pymethods]
+impl Cursor {
+    #[pyo3(signature = (sql, parameters=None))]
+    pub fn execute(&mut self, sql: &str, parameters: Option<Py<PyTuple>>) -> Result<Self> {
+        let stmt_is_dml = stmt_is_dml(sql);
+
+        let conn_lock =
+            self.conn.conn.lock().map_err(|_| {
+                PyErr::new::<OperationalError, _>("Failed to acquire connection lock")
+            })?;
+
+        let statement = conn_lock.prepare(sql).map_err(|e| {
+            PyErr::new::<ProgrammingError, _>(format!("Failed to prepare statement: {:?}", e))
+        })?;
+
+        self.smt = Some(Arc::new(Mutex::new(statement)));
+
+        // TODO: use stmt_is_dml to set rowcount
+        if stmt_is_dml {
+            todo!()
+        }
+
+        Ok(Cursor {
+            smt: self.smt.clone(),
+            conn: self.conn.clone(),
+            description: self.description.clone(),
+            rowcount: self.rowcount,
+            arraysize: self.arraysize,
+        })
+    }
+
+    pub fn fetchone(&mut self, py: Python) -> Result<Option<PyObject>> {
+        if let Some(smt) = &self.smt {
+            let mut smt_lock = smt.lock().map_err(|_| {
+                PyErr::new::<OperationalError, _>("Failed to acquire statement lock")
+            })?;
+
+            match smt_lock
+                .step()
+                .map_err(|e| PyErr::new::<OperationalError, _>(format!("Step error: {:?}", e)))?
+            {
+                limbo_core::RowResult::Row(row) => {
+                    let py_row = row_to_py(py, &row);
+                    Ok(Some(py_row))
+                }
+                limbo_core::RowResult::IO => {
+                    self.conn.io.run_once().map_err(|e| {
+                        PyErr::new::<OperationalError, _>(format!("IO error: {:?}", e))
+                    })?;
+                    Ok(None)
+                }
+                limbo_core::RowResult::Done => Ok(None),
+            }
+        } else {
+            Err(PyErr::new::<ProgrammingError, _>("No statement prepared for execution").into())
+        }
+    }
+
+    pub fn fetchall(&mut self, py: Python) -> Result<Vec<PyObject>> {
+        let mut results = Vec::new();
+        while let Some(row) = self.fetchone(py)? {
+            results.push(row);
+        }
+        Ok(results)
+    }
+
+    pub fn close(&self) -> Result<()> {
+        todo!()
+    }
+
+    #[pyo3(signature = (sql, parameters=None))]
+    pub fn executemany(&self, sql: &str, parameters: Option<Py<PyList>>) {
+        todo!()
+    }
+
+    #[pyo3(signature = (size=None))]
+    pub fn fetchmany(&self, size: Option<i64>) {
+        todo!()
+    }
+}
+
+fn stmt_is_dml(sql: &str) -> bool {
+    let sql = sql.trim();
+    let sql = sql.to_uppercase();
+    sql.starts_with("INSERT") || sql.starts_with("UPDATE") || sql.starts_with("DELETE")
+}
+
+#[pyclass]
+#[derive(Clone)]
+pub struct Connection {
+    conn: Arc<Mutex<limbo_core::Connection>>,
+    io: Arc<limbo_core::PlatformIO>,
+}
+
+// SAFETY: The limbo_core crate guarantees that `Connection` is thread-safe.
+unsafe impl Send for Connection {}
+
+#[pymethods]
+impl Connection {
+    pub fn cursor(&self) -> Result<Cursor> {
+        Ok(Cursor {
+            arraysize: 1,
+            conn: self.clone(),
+            description: None,
+            rowcount: -1,
+            smt: None,
+        })
+    }
+
+    pub fn close(&self) {
+        drop(self.conn.clone());
+    }
+
+    pub fn commit(&self) {
+        todo!()
+    }
+
+    pub fn rollback(&self) {
+        todo!()
+    }
+}
+
+#[pyfunction]
+pub fn connect(path: &str) -> Result<Connection> {
+    let io = Arc::new(limbo_core::PlatformIO::new().map_err(|e| {
+        PyErr::new::<InterfaceError, _>(format!("IO initialization failed: {:?}", e))
+    })?);
+    let db = limbo_core::Database::open_file(io.clone(), path)
+        .map_err(|e| PyErr::new::<DatabaseError, _>(format!("Failed to open database: {:?}", e)))?;
+    let conn: limbo_core::Connection = db.connect();
+    Ok(Connection {
+        conn: Arc::new(Mutex::new(conn)),
+        io,
+    })
+}
+
+fn row_to_py(py: Python, row: &limbo_core::Row) -> PyObject {
+    let py_values: Vec<PyObject> = row
+        .values
+        .iter()
+        .map(|value| match value {
+            limbo_core::Value::Null => py.None(),
+            limbo_core::Value::Integer(i) => i.to_object(py),
+            limbo_core::Value::Float(f) => f.to_object(py),
+            limbo_core::Value::Text(s) => s.to_object(py),
+            limbo_core::Value::Blob(b) => b.to_object(py),
+        })
+        .collect();
+
+    PyTuple::new_bound(py, &py_values).to_object(py)
+}
+
+#[pymodule]
+fn _limbo(m: &Bound<PyModule>) -> PyResult<()> {
+    m.add("__version__", env!("CARGO_PKG_VERSION"))?;
+    m.add_class::<Connection>()?;
+    m.add_class::<Cursor>()?;
+    m.add_function(wrap_pyfunction!(connect, m)?)?;
+    m.add("Warning", m.py().get_type_bound::<Warning>())?;
+    m.add("Error", m.py().get_type_bound::<Error>())?;
+    m.add("InterfaceError", m.py().get_type_bound::<InterfaceError>())?;
+    m.add("DatabaseError", m.py().get_type_bound::<DatabaseError>())?;
+    m.add("DataError", m.py().get_type_bound::<DataError>())?;
+    m.add(
+        "OperationalError",
+        m.py().get_type_bound::<OperationalError>(),
+    )?;
+    m.add("IntegrityError", m.py().get_type_bound::<IntegrityError>())?;
+    m.add("InternalError", m.py().get_type_bound::<InternalError>())?;
+    m.add(
+        "ProgrammingError",
+        m.py().get_type_bound::<ProgrammingError>(),
+    )?;
+    m.add(
+        "NotSupportedError",
+        m.py().get_type_bound::<NotSupportedError>(),
+    )?;
+    Ok(())
+}

bindings/python/tests/test_database.py

@@ -0,0 +1,66 @@
+import sqlite3
+
+import pytest
+
+import limbo
+
+
+@pytest.mark.parametrize("provider", ["sqlite3", "limbo"])
+def test_fetchall_select_all_users(provider):
+    conn = connect(provider, "tests/database.db")
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM users")
+
+    users = cursor.fetchall()
+    assert users
+    assert users == [(1, "alice"), (2, "bob")]
+
+
+@pytest.mark.parametrize(
+    "provider",
+    [
+        "sqlite3",
+    ],
+)
+def test_fetchall_select_user_ids(provider):
+    conn = connect(provider, "tests/database.db")
+    cursor = conn.cursor()
+    cursor.execute("SELECT id FROM users")
+
+    user_ids = cursor.fetchall()
+    assert user_ids
+    assert user_ids == [(1,), (2,)]
+
+
+@pytest.mark.parametrize("provider", ["sqlite3", "limbo"])
+def test_fetchone_select_all_users(provider):
+    conn = connect(provider, "tests/database.db")
+    cursor = conn.cursor()
+    cursor.execute("SELECT * FROM users")
+
+    alice = cursor.fetchone()
+    assert alice
+    assert alice == (1, "alice")
+
+    bob = cursor.fetchone()
+    assert bob
+    assert bob == (2, "bob")
+
+
+@pytest.mark.parametrize("provider", ["sqlite3", "limbo"])
+def test_fetchone_select_max_user_id(provider):
+    conn = connect(provider, "tests/database.db")
+    cursor = conn.cursor()
+    cursor.execute("SELECT MAX(id) FROM users")
+
+    max_id = cursor.fetchone()
+    assert max_id
+    assert max_id == (2,)
+
+
+def connect(provider, database):
+    if provider == "limbo":
+        return limbo.connect(database)
+    if provider == "sqlite3":
+        return sqlite3.connect(database)
+    raise Exception(f"Provider `{provider}` is not supported")