aljaz/cdk
1
0
Fork 0
mirror of https://github.com/aljazceru/cdk.git synced 2025-12-23 15:44:50 +01:00
Code Issues Packages Projects Releases Wiki Activity

Working on a better database abstraction (#931)

Browse Source 
* Working on a better database abstraction

After [this question in the chat](https://matrix.to/#/!oJFtttFHGfnTGrIjvD:matrix.cashu.space/$oJFtttFHGfnTGrIjvD:matrix.cashu.space/$I5ZtjJtBM0ctltThDYpoCwClZFlM6PHzf8q2Rjqmso8)
regarding a database transaction within the same function, I realized a few
design flaws in our SQL database abstraction, particularly regarding
transactions.

1. Our upper abstraction got it right, where a transaction is bound with `&mut
   self`, so Rust knows how to handle its lifetime with' async/await'.
2. The raw database does not; instead, it returns &self, and beginning a
   transaction takes &self as well, which is problematic for Rust, but that's not
   all. It is fundamentally wrong. A transaction should take &mut self when
   beginning a transaction, as that connection is bound to a transaction and
   should not be returned to the pool. Currently, that responsibility lies with
   the implementor. If a mistake is made, a transaction could be executed in two
   or more connections.
3. The way a database is bound to our store layer is through a single struct,
   which may or may not internally utilize our connection pool. This is also
   another design flow, in this PR, a connection pool is owned, and to use a
   connection, it should be requested, and that connection is reference with
   mutable when beginning a transaction

* Improve the abstraction with fewer generics

As suggested by @thesimplekid

* Add BEGIN IMMEDIATE for SQLite
This commit is contained in:
C
2025-08-06 03:58:03 -03:00
committed by GitHub
parent cceea654fe
commit ad8f1ece5c
13 changed files with 721 additions and 1122 deletions
Download Patch File Download Diff File Expand all files Collapse all files

197
crates/cdk-sqlite/src/async_sqlite.rs Normal file
View File

@@ -0,0 +1,197 @@
//! Simple SQLite
use cdk_common::database::Error;
use cdk_sql_common::database::{DatabaseConnector, DatabaseExecutor, DatabaseTransaction};
use cdk_sql_common::stmt::{query, Column, SqlPart, Statement};
use rusqlite::{ffi, CachedStatement, Connection, Error as SqliteError, ErrorCode};
use tokio::sync::Mutex;
use crate::common::{from_sqlite, to_sqlite};
/// Async Sqlite wrapper
#[derive(Debug)]
pub struct AsyncSqlite {
inner: Mutex<Connection>,
}
impl AsyncSqlite {
pub fn new(inner: Connection) -> Self {
Self {
inner: inner.into(),
}
}
}
impl AsyncSqlite {
fn get_stmt<'a>(
&self,
conn: &'a Connection,
statement: Statement,
) -> Result<CachedStatement<'a>, Error> {
let (sql, placeholder_values) = statement.to_sql()?;
let new_sql = sql.trim().trim_end_matches("FOR UPDATE");
let mut stmt = conn
.prepare_cached(new_sql)
.map_err(|e| Error::Database(Box::new(e)))?;
for (i, value) in placeholder_values.into_iter().enumerate() {
stmt.raw_bind_parameter(i + 1, to_sqlite(value))
.map_err(|e| Error::Database(Box::new(e)))?;
}
Ok(stmt)
}
}
#[inline(always)]
fn to_sqlite_error(err: SqliteError) -> Error {
tracing::error!("Failed query with error {:?}", err);
if let rusqlite::Error::SqliteFailure(
ffi::Error {
code,
extended_code,
},
_,
) = err
{
if code == ErrorCode::ConstraintViolation
&& (extended_code == ffi::SQLITE_CONSTRAINT_PRIMARYKEY
|| extended_code == ffi::SQLITE_CONSTRAINT_UNIQUE)
{
Error::Duplicate
} else {
Error::Database(Box::new(err))
}
} else {
Error::Database(Box::new(err))
}
}
/// SQLite trasanction handler
pub struct SQLiteTransactionHandler;
#[async_trait::async_trait]
impl DatabaseTransaction<AsyncSqlite> for SQLiteTransactionHandler {
/// Consumes the current transaction committing the changes
async fn commit(conn: &mut AsyncSqlite) -> Result<(), Error> {
query("COMMIT")?.execute(conn).await?;
Ok(())
}
/// Begin a transaction
async fn begin(conn: &mut AsyncSqlite) -> Result<(), Error> {
query("BEGIN IMMEDIATE")?.execute(conn).await?;
Ok(())
}
/// Consumes the transaction rolling back all changes
async fn rollback(conn: &mut AsyncSqlite) -> Result<(), Error> {
query("ROLLBACK")?.execute(conn).await?;
Ok(())
}
}
impl DatabaseConnector for AsyncSqlite {
type Transaction = SQLiteTransactionHandler;
}
#[async_trait::async_trait]
impl DatabaseExecutor for AsyncSqlite {
fn name() -> &'static str {
"sqlite"
}
async fn execute(&self, statement: Statement) -> Result<usize, Error> {
let conn = self.inner.lock().await;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
Ok(stmt.raw_execute().map_err(to_sqlite_error)?)
}
async fn fetch_one(&self, statement: Statement) -> Result<Option<Vec<Column>>, Error> {
let conn = self.inner.lock().await;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
let columns = stmt.column_count();
let mut rows = stmt.raw_query();
rows.next()
.map_err(to_sqlite_error)?
.map(|row| {
(0..columns)
.map(|i| row.get(i).map(from_sqlite))
.collect::<Result<Vec<_>, _>>()
})
.transpose()
.map_err(to_sqlite_error)
}
async fn fetch_all(&self, statement: Statement) -> Result<Vec<Vec<Column>>, Error> {
let conn = self.inner.lock().await;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
let columns = stmt.column_count();
let mut rows = stmt.raw_query();
let mut results = vec![];
while let Some(row) = rows.next().map_err(to_sqlite_error)? {
results.push(
(0..columns)
.map(|i| row.get(i).map(from_sqlite))
.collect::<Result<Vec<_>, _>>()
.map_err(to_sqlite_error)?,
)
}
Ok(results)
}
async fn pluck(&self, statement: Statement) -> Result<Option<Column>, Error> {
let conn = self.inner.lock().await;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
let mut rows = stmt.raw_query();
rows.next()
.map_err(to_sqlite_error)?
.map(|row| row.get(0usize).map(from_sqlite))
.transpose()
.map_err(to_sqlite_error)
}
async fn batch(&self, mut statement: Statement) -> Result<(), Error> {
let sql = {
let part = statement
.parts
.pop()
.ok_or(Error::Internal("Empty SQL".to_owned()))?;
if !statement.parts.is_empty() || matches!(part, SqlPart::Placeholder(_, _)) {
return Err(Error::Internal(
"Invalid usage, batch does not support placeholders".to_owned(),
));
}
if let SqlPart::Raw(sql) = part {
sql
} else {
unreachable!()
}
};
self.inner
.lock()
.await
.execute_batch(&sql)
.map_err(to_sqlite_error)
}
}

90
crates/cdk-sqlite/src/common.rs
View File

@@ -1,11 +1,14 @@
use std::path::PathBuf;
use std::sync::atomic::AtomicBool;
use std::sync::Arc;
use std::time::Duration;
use cdk_sql_common::pool::{self, Pool, ResourceManager};
use cdk_sql_common::pool::{self, DatabasePool};
use cdk_sql_common::value::Value;
use rusqlite::Connection;
use crate::async_sqlite;
/// The config need to create a new SQLite connection
#[derive(Clone, Debug)]
pub struct Config {
@@ -13,14 +16,28 @@ pub struct Config {
password: Option<String>,
}
impl pool::DatabaseConfig for Config {
fn default_timeout(&self) -> Duration {
Duration::from_secs(5)
}
fn max_size(&self) -> usize {
if self.password.is_none() {
1
} else {
20
}
}
}
/// Sqlite connection manager
#[derive(Debug)]
pub struct SqliteConnectionManager;
impl ResourceManager for SqliteConnectionManager {
impl DatabasePool for SqliteConnectionManager {
type Config = Config;
type Resource = Connection;
type Connection = async_sqlite::AsyncSqlite;
type Error = rusqlite::Error;
@@ -28,7 +45,7 @@ impl ResourceManager for SqliteConnectionManager {
config: &Self::Config,
_stale: Arc<AtomicBool>,
_timeout: Duration,
) -> Result<Self::Resource, pool::Error<Self::Error>> {
) -> Result<Self::Connection, pool::Error<Self::Error>> {
let conn = if let Some(path) = config.path.as_ref() {
Connection::open(path)?
} else {
@@ -52,35 +69,58 @@ impl ResourceManager for SqliteConnectionManager {
conn.busy_timeout(Duration::from_secs(10))?;
Ok(conn)
Ok(async_sqlite::AsyncSqlite::new(conn))
}
}
/// Create a configured rusqlite connection to a SQLite database.
/// For SQLCipher support, enable the "sqlcipher" feature and pass a password.
pub fn create_sqlite_pool(
path: &str,
password: Option<String>,
) -> Arc<Pool<SqliteConnectionManager>> {
let (config, max_size) = if path.contains(":memory:") {
(
impl From<PathBuf> for Config {
fn from(path: PathBuf) -> Self {
path.to_str().unwrap_or_default().into()
}
}
impl From<(PathBuf, String)> for Config {
fn from((path, password): (PathBuf, String)) -> Self {
(path.to_str().unwrap_or_default(), password.as_str()).into()
}
}
impl From<&PathBuf> for Config {
fn from(path: &PathBuf) -> Self {
path.to_str().unwrap_or_default().into()
}
}
impl From<&str> for Config {
fn from(path: &str) -> Self {
if path.contains(":memory:") {
Config {
path: None,
password,
},
1,
)
} else {
(
password: None,
}
} else {
Config {
path: Some(path.to_owned()),
password,
},
20,
)
};
password: None,
}
}
}
}
Pool::new(config, max_size, Duration::from_secs(10))
impl From<(&str, &str)> for Config {
fn from((path, pass): (&str, &str)) -> Self {
if path.contains(":memory:") {
Config {
path: None,
password: Some(pass.to_owned()),
}
} else {
Config {
path: Some(path.to_owned()),
password: Some(pass.to_owned()),
}
}
}
}
/// Convert cdk_sql_common::value::Value to rusqlite Value

1
crates/cdk-sqlite/src/lib.rs
View File

@@ -3,6 +3,7 @@
#![warn(missing_docs)]
#![warn(rustdoc::bare_urls)]
mod async_sqlite;
mod common;
#[cfg(feature = "mint")]

727
crates/cdk-sqlite/src/mint/async_rusqlite.rs
View File

@@ -1,727 +0,0 @@
//! Async, pipelined rusqlite client
use std::marker::PhantomData;
use std::path::PathBuf;
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{mpsc as std_mpsc, Arc, Mutex};
use std::thread::spawn;
use std::time::Instant;
use cdk_common::database::Error;
use cdk_sql_common::database::{DatabaseConnector, DatabaseExecutor, DatabaseTransaction};
use cdk_sql_common::pool::{self, Pool, PooledResource};
use cdk_sql_common::stmt::{Column, ExpectedSqlResponse, Statement as InnerStatement};
use cdk_sql_common::ConversionError;
use rusqlite::{ffi, Connection, ErrorCode, TransactionBehavior};
use tokio::sync::{mpsc, oneshot};
use crate::common::{create_sqlite_pool, from_sqlite, to_sqlite, SqliteConnectionManager};
/// The number of queued SQL statements before it start failing
const SQL_QUEUE_SIZE: usize = 10_000;
/// How many ms is considered a slow query, and it'd be logged for further debugging
const SLOW_QUERY_THRESHOLD_MS: u128 = 20;
/// How many SQLite parallel connections can be used to read things in parallel
const WORKING_THREAD_POOL_SIZE: usize = 5;
#[derive(Debug, Clone)]
pub struct AsyncRusqlite {
sender: mpsc::Sender<DbRequest>,
inflight_requests: Arc<AtomicUsize>,
}
impl From<PathBuf> for AsyncRusqlite {
fn from(value: PathBuf) -> Self {
AsyncRusqlite::new(create_sqlite_pool(value.to_str().unwrap_or_default(), None))
}
}
impl From<&str> for AsyncRusqlite {
fn from(value: &str) -> Self {
AsyncRusqlite::new(create_sqlite_pool(value, None))
}
}
impl From<(&str, &str)> for AsyncRusqlite {
fn from((value, pass): (&str, &str)) -> Self {
AsyncRusqlite::new(create_sqlite_pool(value, Some(pass.to_owned())))
}
}
impl From<(PathBuf, &str)> for AsyncRusqlite {
fn from((value, pass): (PathBuf, &str)) -> Self {
AsyncRusqlite::new(create_sqlite_pool(
value.to_str().unwrap_or_default(),
Some(pass.to_owned()),
))
}
}
impl From<(&str, String)> for AsyncRusqlite {
fn from((value, pass): (&str, String)) -> Self {
AsyncRusqlite::new(create_sqlite_pool(value, Some(pass)))
}
}
impl From<(PathBuf, String)> for AsyncRusqlite {
fn from((value, pass): (PathBuf, String)) -> Self {
AsyncRusqlite::new(create_sqlite_pool(
value.to_str().unwrap_or_default(),
Some(pass),
))
}
}
impl From<&PathBuf> for AsyncRusqlite {
fn from(value: &PathBuf) -> Self {
AsyncRusqlite::new(create_sqlite_pool(value.to_str().unwrap_or_default(), None))
}
}
/// Internal request for the database thread
#[derive(Debug)]
enum DbRequest {
Sql(InnerStatement, oneshot::Sender<DbResponse>),
Begin(oneshot::Sender<DbResponse>),
Commit(oneshot::Sender<DbResponse>),
Rollback(oneshot::Sender<DbResponse>),
}
#[derive(Debug)]
enum DbResponse {
Transaction(mpsc::Sender<DbRequest>),
AffectedRows(usize),
Pluck(Option<Column>),
Row(Option<Vec<Column>>),
Rows(Vec<Vec<Column>>),
Error(SqliteError),
Unexpected,
Ok,
}
#[derive(thiserror::Error, Debug)]
enum SqliteError {
#[error(transparent)]
Sqlite(#[from] rusqlite::Error),
#[error(transparent)]
Inner(#[from] Error),
#[error(transparent)]
Pool(#[from] pool::Error<rusqlite::Error>),
/// Duplicate entry
#[error("Duplicate")]
Duplicate,
#[error(transparent)]
Conversion(#[from] ConversionError),
}
impl From<SqliteError> for Error {
fn from(val: SqliteError) -> Self {
match val {
SqliteError::Duplicate => Error::Duplicate,
SqliteError::Conversion(e) => e.into(),
o => Error::Internal(o.to_string()),
}
}
}
/// Process a query
#[inline(always)]
fn process_query(conn: &Connection, statement: InnerStatement) -> Result<DbResponse, SqliteError> {
let start = Instant::now();
let expected_response = statement.expected_response;
let (sql, placeholder_values) = statement.to_sql()?;
let sql = sql.trim_end_matches("FOR UPDATE");
let mut stmt = conn.prepare_cached(sql)?;
for (i, value) in placeholder_values.into_iter().enumerate() {
stmt.raw_bind_parameter(i + 1, to_sqlite(value))?;
}
let columns = stmt.column_count();
let to_return = match expected_response {
ExpectedSqlResponse::AffectedRows => DbResponse::AffectedRows(stmt.raw_execute()?),
ExpectedSqlResponse::Batch => {
conn.execute_batch(sql)?;
DbResponse::Ok
}
ExpectedSqlResponse::ManyRows => {
let mut rows = stmt.raw_query();
let mut results = vec![];
while let Some(row) = rows.next()? {
results.push(
(0..columns)
.map(|i| row.get(i).map(from_sqlite))
.collect::<Result<Vec<_>, _>>()?,
)
}
DbResponse::Rows(results)
}
ExpectedSqlResponse::Pluck => {
let mut rows = stmt.raw_query();
DbResponse::Pluck(
rows.next()?
.map(|row| row.get(0usize).map(from_sqlite))
.transpose()?,
)
}
ExpectedSqlResponse::SingleRow => {
let mut rows = stmt.raw_query();
let row = rows
.next()?
.map(|row| {
(0..columns)
.map(|i| row.get(i).map(from_sqlite))
.collect::<Result<Vec<_>, _>>()
})
.transpose()?;
DbResponse::Row(row)
}
};
let duration = start.elapsed();
if duration.as_millis() > SLOW_QUERY_THRESHOLD_MS {
tracing::warn!("[SLOW QUERY] Took {} ms: {}", duration.as_millis(), sql);
}
Ok(to_return)
}
/// Spawns N number of threads to execute SQL statements
///
/// Enable parallelism with a pool of threads.
///
/// There is a main thread, which receives SQL requests and routes them to a worker thread from a
/// fixed-size pool.
///
/// By doing so, SQLite does synchronization, and Rust will only intervene when a transaction is
/// executed. Transactions are executed in the main thread.
fn rusqlite_spawn_worker_threads(
inflight_requests: Arc<AtomicUsize>,
threads: usize,
) -> std_mpsc::Sender<(
PooledResource<SqliteConnectionManager>,
InnerStatement,
oneshot::Sender<DbResponse>,
)> {
let (sender, receiver) = std_mpsc::channel::<(
PooledResource<SqliteConnectionManager>,
InnerStatement,
oneshot::Sender<DbResponse>,
)>();
let receiver = Arc::new(Mutex::new(receiver));
for _ in 0..threads {
let rx = receiver.clone();
let inflight_requests = inflight_requests.clone();
spawn(move || loop {
while let Ok((conn, sql, reply_to)) = rx.lock().expect("failed to acquire").recv() {
let result = process_query(&conn, sql);
let _ = match result {
Ok(ok) => reply_to.send(ok),
Err(err) => {
tracing::error!("Failed query with error {:?}", err);
let err = if let SqliteError::Sqlite(rusqlite::Error::SqliteFailure(
ffi::Error {
code,
extended_code,
},
_,
)) = &err
{
if *code == ErrorCode::ConstraintViolation
&& (*extended_code == ffi::SQLITE_CONSTRAINT_PRIMARYKEY
|| *extended_code == ffi::SQLITE_CONSTRAINT_UNIQUE)
{
SqliteError::Duplicate
} else {
err
}
} else {
err
};
reply_to.send(DbResponse::Error(err))
}
};
drop(conn);
inflight_requests.fetch_sub(1, Ordering::Relaxed);
}
});
}
sender
}
/// # Rusqlite main worker
///
/// This function takes ownership of a pool of connections to SQLite, executes SQL statements, and
/// returns the results or number of affected rows to the caller. All communications are done
/// through channels. This function is synchronous, but a thread pool exists to execute queries, and
/// SQLite will coordinate data access. Transactions are executed in the main and it takes ownership
/// of the main thread until it is finalized
///
/// This is meant to be called in their thread, as it will not exit the loop until the communication
/// channel is closed.
fn rusqlite_worker_manager(
mut receiver: mpsc::Receiver<DbRequest>,
pool: Arc<Pool<SqliteConnectionManager>>,
inflight_requests: Arc<AtomicUsize>,
) {
let send_sql_to_thread =
rusqlite_spawn_worker_threads(inflight_requests.clone(), WORKING_THREAD_POOL_SIZE);
let mut tx_id: usize = 0;
while let Some(request) = receiver.blocking_recv() {
inflight_requests.fetch_add(1, Ordering::Relaxed);
match request {
DbRequest::Sql(statement, reply_to) => {
let conn = match pool.get() {
Ok(conn) => conn,
Err(err) => {
tracing::error!("Failed to acquire a pool connection: {:?}", err);
inflight_requests.fetch_sub(1, Ordering::Relaxed);
let _ = reply_to.send(DbResponse::Error(err.into()));
continue;
}
};
let _ = send_sql_to_thread.send((conn, statement, reply_to));
continue;
}
DbRequest::Begin(reply_to) => {
let (sender, mut receiver) = mpsc::channel(SQL_QUEUE_SIZE);
let mut conn = match pool.get() {
Ok(conn) => conn,
Err(err) => {
tracing::error!("Failed to acquire a pool connection: {:?}", err);
inflight_requests.fetch_sub(1, Ordering::Relaxed);
let _ = reply_to.send(DbResponse::Error(err.into()));
continue;
}
};
let tx = match conn.transaction_with_behavior(TransactionBehavior::Immediate) {
Ok(tx) => tx,
Err(err) => {
tracing::error!("Failed to begin a transaction: {:?}", err);
inflight_requests.fetch_sub(1, Ordering::Relaxed);
let _ = reply_to.send(DbResponse::Error(err.into()));
continue;
}
};
// Transaction has begun successfully, send the `sender` back to the caller
// and wait for statements to execute. On `Drop` the wrapper transaction
// should send a `rollback`.
let _ = reply_to.send(DbResponse::Transaction(sender));
tx_id += 1;
// We intentionally handle the transaction hijacking the main loop, there is
// no point is queueing more operations for SQLite, since transaction have
// exclusive access. In other database implementation this block of code
// should be sent to their own thread to allow concurrency
loop {
let request = if let Some(request) = receiver.blocking_recv() {
request
} else {
// If the receiver loop is broken (i.e no more `senders` are active) and no
// `Commit` statement has been sent, this will trigger a `Rollback`
// automatically
tracing::trace!("Tx {}: Transaction rollback on drop", tx_id);
let _ = tx.rollback();
break;
};
match request {
DbRequest::Commit(reply_to) => {
tracing::trace!("Tx {}: Commit", tx_id);
let _ = reply_to.send(match tx.commit() {
Ok(()) => DbResponse::Ok,
Err(err) => {
tracing::error!("Failed commit {:?}", err);
DbResponse::Error(err.into())
}
});
break;
}
DbRequest::Rollback(reply_to) => {
tracing::trace!("Tx {}: Rollback", tx_id);
let _ = reply_to.send(match tx.rollback() {
Ok(()) => DbResponse::Ok,
Err(err) => {
tracing::error!("Failed rollback {:?}", err);
DbResponse::Error(err.into())
}
});
break;
}
DbRequest::Begin(reply_to) => {
let _ = reply_to.send(DbResponse::Unexpected);
}
DbRequest::Sql(statement, reply_to) => {
tracing::trace!("Tx {}: SQL {:?}", tx_id, statement);
let _ = match process_query(&tx, statement) {
Ok(ok) => reply_to.send(ok),
Err(err) => {
tracing::error!(
"Tx {}: Failed query with error {:?}",
tx_id,
err
);
let err = if let SqliteError::Sqlite(
rusqlite::Error::SqliteFailure(
ffi::Error {
code,
extended_code,
},
_,
),
) = &err
{
if *code == ErrorCode::ConstraintViolation
&& (*extended_code == ffi::SQLITE_CONSTRAINT_PRIMARYKEY
|| *extended_code == ffi::SQLITE_CONSTRAINT_UNIQUE)
{
SqliteError::Duplicate
} else {
err
}
} else {
err
};
reply_to.send(DbResponse::Error(err))
}
};
}
}
}
drop(conn);
}
DbRequest::Commit(reply_to) => {
let _ = reply_to.send(DbResponse::Unexpected);
}
DbRequest::Rollback(reply_to) => {
let _ = reply_to.send(DbResponse::Unexpected);
}
}
// If wasn't a `continue` the transaction is done by reaching this code, and we should
// decrease the inflight_request counter
inflight_requests.fetch_sub(1, Ordering::Relaxed);
}
}
impl AsyncRusqlite {
/// Creates a new Async Rusqlite wrapper.
pub fn new(pool: Arc<Pool<SqliteConnectionManager>>) -> Self {
let (sender, receiver) = mpsc::channel(SQL_QUEUE_SIZE);
let inflight_requests = Arc::new(AtomicUsize::new(0));
let inflight_requests_for_thread = inflight_requests.clone();
spawn(move || {
rusqlite_worker_manager(receiver, pool, inflight_requests_for_thread);
});
Self {
sender,
inflight_requests,
}
}
fn get_queue_sender(&self) -> &mpsc::Sender<DbRequest> {
&self.sender
}
/// Show how many inflight requests
#[allow(dead_code)]
pub fn inflight_requests(&self) -> usize {
self.inflight_requests.load(Ordering::Relaxed)
}
}
#[async_trait::async_trait]
impl DatabaseConnector for AsyncRusqlite {
type Transaction<'a> = Transaction<'a>;
/// Begins a transaction
///
/// If the transaction is Drop it will trigger a rollback operation
async fn begin(&self) -> Result<Self::Transaction<'_>, Error> {
let (sender, receiver) = oneshot::channel();
self.sender
.send(DbRequest::Begin(sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Transaction(db_sender) => Ok(Transaction {
db_sender,
_marker: PhantomData,
}),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
}
#[async_trait::async_trait]
impl DatabaseExecutor for AsyncRusqlite {
fn name() -> &'static str {
"sqlite"
}
async fn fetch_one(&self, mut statement: InnerStatement) -> Result<Option<Vec<Column>>, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::SingleRow;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Row(row) => Ok(row),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn batch(&self, mut statement: InnerStatement) -> Result<(), Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::Batch;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Ok => Ok(()),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn fetch_all(&self, mut statement: InnerStatement) -> Result<Vec<Vec<Column>>, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::ManyRows;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Rows(row) => Ok(row),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn execute(&self, mut statement: InnerStatement) -> Result<usize, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::AffectedRows;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::AffectedRows(total) => Ok(total),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn pluck(&self, mut statement: InnerStatement) -> Result<Option<Column>, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::Pluck;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Pluck(value) => Ok(value),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
}
/// Database transaction
#[derive(Debug)]
pub struct Transaction<'conn> {
db_sender: mpsc::Sender<DbRequest>,
_marker: PhantomData<&'conn ()>,
}
impl Transaction<'_> {
fn get_queue_sender(&self) -> &mpsc::Sender<DbRequest> {
&self.db_sender
}
}
impl Drop for Transaction<'_> {
fn drop(&mut self) {
let (sender, _) = oneshot::channel();
let _ = self.db_sender.try_send(DbRequest::Rollback(sender));
}
}
#[async_trait::async_trait]
impl<'a> DatabaseTransaction<'a> for Transaction<'a> {
async fn commit(self) -> Result<(), Error> {
let (sender, receiver) = oneshot::channel();
self.db_sender
.send(DbRequest::Commit(sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Ok => Ok(()),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn rollback(self) -> Result<(), Error> {
let (sender, receiver) = oneshot::channel();
self.db_sender
.send(DbRequest::Rollback(sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Ok => Ok(()),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
}
#[async_trait::async_trait]
impl DatabaseExecutor for Transaction<'_> {
fn name() -> &'static str {
"sqlite"
}
async fn fetch_one(&self, mut statement: InnerStatement) -> Result<Option<Vec<Column>>, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::SingleRow;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Row(row) => Ok(row),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn batch(&self, mut statement: InnerStatement) -> Result<(), Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::Batch;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Ok => Ok(()),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn fetch_all(&self, mut statement: InnerStatement) -> Result<Vec<Vec<Column>>, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::ManyRows;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Rows(row) => Ok(row),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn execute(&self, mut statement: InnerStatement) -> Result<usize, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::AffectedRows;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::AffectedRows(total) => Ok(total),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
async fn pluck(&self, mut statement: InnerStatement) -> Result<Option<Column>, Error> {
let (sender, receiver) = oneshot::channel();
statement.expected_response = ExpectedSqlResponse::Pluck;
self.get_queue_sender()
.send(DbRequest::Sql(statement, sender))
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?;
match receiver
.await
.map_err(|_| Error::Internal("Communication".to_owned()))?
{
DbResponse::Pluck(value) => Ok(value),
DbResponse::Error(err) => Err(err.into()),
_ => Err(Error::InvalidDbResponse),
}
}
}

21
crates/cdk-sqlite/src/mint/mod.rs
View File

@@ -3,26 +3,27 @@
use cdk_sql_common::mint::SQLMintAuthDatabase;
use cdk_sql_common::SQLMintDatabase;
mod async_rusqlite;
use crate::common::SqliteConnectionManager;
pub mod memory;
/// Mint SQLite implementation with rusqlite
pub type MintSqliteDatabase = SQLMintDatabase<async_rusqlite::AsyncRusqlite>;
pub type MintSqliteDatabase = SQLMintDatabase<SqliteConnectionManager>;
/// Mint Auth database with rusqlite
#[cfg(feature = "auth")]
pub type MintSqliteAuthDatabase = SQLMintAuthDatabase<async_rusqlite::AsyncRusqlite>;
pub type MintSqliteAuthDatabase = SQLMintAuthDatabase<SqliteConnectionManager>;
#[cfg(test)]
mod test {
use std::fs::remove_file;
use cdk_common::mint_db_test;
use cdk_sql_common::pool::Pool;
use cdk_sql_common::stmt::query;
use super::*;
use crate::mint::async_rusqlite::AsyncRusqlite;
use crate::common::Config;
async fn provide_db() -> MintSqliteDatabase {
memory::empty().await.unwrap()
@@ -40,13 +41,17 @@ mod test {
{
let _ = remove_file(&file);
#[cfg(not(feature = "sqlcipher"))]
let conn: AsyncRusqlite = file.as_str().into();
let config: Config = file.as_str().into();
#[cfg(feature = "sqlcipher")]
let conn: AsyncRusqlite = (file.as_str(), "test".to_owned()).into();
let config: Config = (file.as_str(), "test").into();
let pool = Pool::<SqliteConnectionManager>::new(config);
let conn = pool.get().expect("valid connection");
query(include_str!("../../tests/legacy-sqlx.sql"))
.expect("query")
.execute(&conn)
.execute(&*conn)
.await
.expect("create former db failed");
}
@@ -55,7 +60,7 @@ mod test {
let conn = MintSqliteDatabase::new(file.as_str()).await;
#[cfg(feature = "sqlcipher")]
let conn = MintSqliteDatabase::new((file.as_str(), "test".to_owned())).await;
let conn = MintSqliteDatabase::new((file.as_str(), "test")).await;
assert!(conn.is_ok(), "Failed with {:?}", conn.unwrap_err());

188
crates/cdk-sqlite/src/wallet/mod.rs
View File

@@ -1,195 +1,13 @@
//! SQLite Wallet Database
use std::path::PathBuf;
use std::sync::Arc;
use cdk_common::database::Error;
use cdk_sql_common::database::DatabaseExecutor;
use cdk_sql_common::pool::{Pool, PooledResource};
use cdk_sql_common::stmt::{Column, SqlPart, Statement};
use cdk_sql_common::SQLWalletDatabase;
use rusqlite::CachedStatement;
use crate::common::{create_sqlite_pool, from_sqlite, to_sqlite, SqliteConnectionManager};
use crate::common::SqliteConnectionManager;
pub mod memory;
/// Simple Sqlite wapper, since the wallet may not need rusqlite with concurrency, a shared instance
/// may be enough
#[derive(Debug)]
pub struct SimpleAsyncRusqlite(Arc<Pool<SqliteConnectionManager>>);
impl SimpleAsyncRusqlite {
fn get_stmt<'a>(
&self,
conn: &'a PooledResource<SqliteConnectionManager>,
statement: Statement,
) -> Result<CachedStatement<'a>, Error> {
let (sql, placeholder_values) = statement.to_sql()?;
let mut stmt = conn
.prepare_cached(&sql)
.map_err(|e| Error::Database(Box::new(e)))?;
for (i, value) in placeholder_values.into_iter().enumerate() {
stmt.raw_bind_parameter(i + 1, to_sqlite(value))
.map_err(|e| Error::Database(Box::new(e)))?;
}
Ok(stmt)
}
}
#[async_trait::async_trait]
impl DatabaseExecutor for SimpleAsyncRusqlite {
fn name() -> &'static str {
"sqlite"
}
async fn execute(&self, statement: Statement) -> Result<usize, Error> {
let conn = self.0.get().map_err(|e| Error::Database(Box::new(e)))?;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
Ok(stmt
.raw_execute()
.map_err(|e| Error::Database(Box::new(e)))?)
}
async fn fetch_one(&self, statement: Statement) -> Result<Option<Vec<Column>>, Error> {
let conn = self.0.get().map_err(|e| Error::Database(Box::new(e)))?;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
let columns = stmt.column_count();
let mut rows = stmt.raw_query();
rows.next()
.map_err(|e| Error::Database(Box::new(e)))?
.map(|row| {
(0..columns)
.map(|i| row.get(i).map(from_sqlite))
.collect::<Result<Vec<_>, _>>()
})
.transpose()
.map_err(|e| Error::Database(Box::new(e)))
}
async fn fetch_all(&self, statement: Statement) -> Result<Vec<Vec<Column>>, Error> {
let conn = self.0.get().map_err(|e| Error::Database(Box::new(e)))?;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
let columns = stmt.column_count();
let mut rows = stmt.raw_query();
let mut results = vec![];
while let Some(row) = rows.next().map_err(|e| Error::Database(Box::new(e)))? {
results.push(
(0..columns)
.map(|i| row.get(i).map(from_sqlite))
.collect::<Result<Vec<_>, _>>()
.map_err(|e| Error::Database(Box::new(e)))?,
)
}
Ok(results)
}
async fn pluck(&self, statement: Statement) -> Result<Option<Column>, Error> {
let conn = self.0.get().map_err(|e| Error::Database(Box::new(e)))?;
let mut stmt = self
.get_stmt(&conn, statement)
.map_err(|e| Error::Database(Box::new(e)))?;
let mut rows = stmt.raw_query();
rows.next()
.map_err(|e| Error::Database(Box::new(e)))?
.map(|row| row.get(0usize).map(from_sqlite))
.transpose()
.map_err(|e| Error::Database(Box::new(e)))
}
async fn batch(&self, mut statement: Statement) -> Result<(), Error> {
let conn = self.0.get().map_err(|e| Error::Database(Box::new(e)))?;
let sql = {
let part = statement
.parts
.pop()
.ok_or(Error::Internal("Empty SQL".to_owned()))?;
if !statement.parts.is_empty() || matches!(part, SqlPart::Placeholder(_, _)) {
return Err(Error::Internal(
"Invalid usage, batch does not support placeholders".to_owned(),
));
}
if let SqlPart::Raw(sql) = part {
sql
} else {
unreachable!()
}
};
conn.execute_batch(&sql)
.map_err(|e| Error::Database(Box::new(e)))
}
}
impl From<PathBuf> for SimpleAsyncRusqlite {
fn from(value: PathBuf) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(value.to_str().unwrap_or_default(), None))
}
}
impl From<&str> for SimpleAsyncRusqlite {
fn from(value: &str) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(value, None))
}
}
impl From<(&str, &str)> for SimpleAsyncRusqlite {
fn from((value, pass): (&str, &str)) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(value, Some(pass.to_owned())))
}
}
impl From<(PathBuf, &str)> for SimpleAsyncRusqlite {
fn from((value, pass): (PathBuf, &str)) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(
value.to_str().unwrap_or_default(),
Some(pass.to_owned()),
))
}
}
impl From<(&str, String)> for SimpleAsyncRusqlite {
fn from((value, pass): (&str, String)) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(value, Some(pass)))
}
}
impl From<(PathBuf, String)> for SimpleAsyncRusqlite {
fn from((value, pass): (PathBuf, String)) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(
value.to_str().unwrap_or_default(),
Some(pass),
))
}
}
impl From<&PathBuf> for SimpleAsyncRusqlite {
fn from(value: &PathBuf) -> Self {
SimpleAsyncRusqlite(create_sqlite_pool(value.to_str().unwrap_or_default(), None))
}
}
/// Mint SQLite implementation with rusqlite
pub type WalletSqliteDatabase = SQLWalletDatabase<SimpleAsyncRusqlite>;
pub type WalletSqliteDatabase = SQLWalletDatabase<SqliteConnectionManager>;
#[cfg(test)]
mod tests {
@@ -330,7 +148,7 @@ mod tests {
// Test PaymentMethod variants
let mint_url = MintUrl::from_str("https://example.com").unwrap();
let payment_methods = vec![
let payment_methods = [
PaymentMethod::Bolt11,
PaymentMethod::Bolt12,
PaymentMethod::Custom("custom".to_string()),