aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2025-12-29 05:54:21 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'cli: Introduce a selectable IO backend with --io={syscall,io-uring} argument' from Jorge López Tello

Browse Source 
Give the user the option of choosing IO backend. There is only a
"syscall" backend, unless built for Linux with `#[cfg(feature =
"io_uring")]` which introduces `--io=io-uring`.
Right now, the choice has only been implemented for the CLI. Bindings
and such default to PlatformIO, except when an "in-memory" database has
been chosen.
Can be tested by running CLI with RUST_LOG=debug

Reviewed-by: Preston Thorpe <preston@unlockedlabs.org>

Closes #654
This commit is contained in:
Pekka Enberg
2025-01-13 20:59:20 +02:00
parent 481bd17219 7365bee984
commit 42293115de
10 changed files with 107 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
cli/Cargo.toml
View File

@@ -29,3 +29,6 @@ rustyline = "12.0.0"
ctrlc = "3.4.4"
csv = "1.3.1"
miette = { version = "7.4.0", features = ["fancy"] }
[features]
io_uring = ["limbo_core/io_uring"]

108
cli/app.rs
View File

@@ -38,6 +38,52 @@ pub struct Opts {
pub quiet: bool,
#[clap(short, long, help = "Print commands before execution")]
pub echo: bool,
#[clap(
default_value_t,
value_enum,
short,
long,
help = "Select I/O backend. The only other choice to 'syscall' is\n\
\t'io-uring' when built for Linux with feature 'io_uring'\n"
)]
pub io: Io,
}
#[derive(Copy, Clone)]
pub enum DbLocation {
Memory,
Path,
}
#[derive(Copy, Clone, ValueEnum)]
pub enum Io {
Syscall,
#[cfg(all(target_os = "linux", feature = "io_uring"))]
IoUring,
}
impl Default for Io {
/// Custom Default impl with cfg! macro, to provide compile-time default to Clap based on platform
/// The cfg! could be elided, but Clippy complains
/// The default value can still be overridden with the Clap argument
fn default() -> Self {
match cfg!(all(target_os = "linux", feature = "io_uring")) {
true => {
#[cfg(all(target_os = "linux", feature = "io_uring"))]
{
Io::IoUring
}
#[cfg(any(
not(target_os = "linux"),
all(target_os = "linux", not(feature = "io_uring"))
))]
{
Io::Syscall
}
}
false => Io::Syscall,
}
}
}
#[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
@@ -160,6 +206,7 @@ pub struct Settings {
output_mode: OutputMode,
echo: bool,
is_stdout: bool,
io: Io,
}
impl From<&Opts> for Settings {
@@ -174,6 +221,7 @@ impl From<&Opts> for Settings {
.database
.as_ref()
.map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string()),
io: opts.io,
}
}
}
@@ -207,7 +255,12 @@ impl Limbo {
.as_ref()
.map_or(":memory:".to_string(), |p| p.to_string_lossy().to_string());
let io = get_io(&db_file)?;
let io = {
match db_file.as_str() {
":memory:" => get_io(DbLocation::Memory, opts.io)?,
_path => get_io(DbLocation::Path, opts.io)?,
}
};
let db = Database::open_file(io.clone(), &db_file)?;
let conn = db.connect();
let interrupt_count = Arc::new(AtomicUsize::new(0));
@@ -293,24 +346,17 @@ impl Limbo {
fn open_db(&mut self, path: &str) -> anyhow::Result<()> {
self.conn.close()?;
match path {
":memory:" => {
let io: Arc<dyn limbo_core::IO> = Arc::new(limbo_core::MemoryIO::new()?);
self.io = Arc::clone(&io);
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = ":memory:".to_string();
Ok(())
let io = {
match path {
":memory:" => get_io(DbLocation::Memory, self.opts.io)?,
_path => get_io(DbLocation::Path, self.opts.io)?,
}
path => {
let io: Arc<dyn limbo_core::IO> = Arc::new(limbo_core::PlatformIO::new()?);
self.io = Arc::clone(&io);
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = path.to_string();
Ok(())
}
}
};
self.io = Arc::clone(&io);
let db = Database::open_file(self.io.clone(), path)?;
self.conn = db.connect();
self.opts.db_file = path.to_string();
Ok(())
}
fn set_output_file(&mut self, path: &str) -> Result<(), String> {
@@ -740,10 +786,28 @@ fn get_writer(output: &str) -> Box<dyn Write> {
}
}
fn get_io(db: &str) -> anyhow::Result<Arc<dyn limbo_core::IO>> {
Ok(match db {
":memory:" => Arc::new(limbo_core::MemoryIO::new()?),
_ => Arc::new(limbo_core::PlatformIO::new()?),
fn get_io(db_location: DbLocation, io_choice: Io) -> anyhow::Result<Arc<dyn limbo_core::IO>> {
Ok(match db_location {
DbLocation::Memory => Arc::new(limbo_core::MemoryIO::new()?),
DbLocation::Path => {
match io_choice {
Io::Syscall => {
// We are building for Linux/macOS and syscall backend has been selected
#[cfg(target_family = "unix")]
{
Arc::new(limbo_core::UnixIO::new()?)
}
// We are not building for Linux/macOS and syscall backend has been selected
#[cfg(not(target_family = "unix"))]
{
Arc::new(limbo_core::PlatformIO::new()?)
}
}
// We are building for Linux and io_uring backend has been selected
#[cfg(all(target_os = "linux", feature = "io_uring"))]
Io::IoUring => Arc::new(limbo_core::UringIO::new()?),
}
}
})
}

2
core/Cargo.toml
View File

@@ -50,7 +50,7 @@ regex-syntax = { version = "0.8.5", default-features = false, features = ["unico
chrono = "0.4.38"
julian_day_converter = "0.3.2"
jsonb = { version = "0.4.4", optional = true }
indexmap = { version="2.2.6", features = ["serde"] }
indexmap = { version = "2.2.6", features = ["serde"] }
serde = { version = "1.0", features = ["derive"] }
pest = { version = "2.0", optional = true }
pest_derive = { version = "2.0", optional = true }

3
core/io/generic.rs
View File

@@ -1,5 +1,5 @@
use crate::{Completion, File, LimboError, OpenFlags, Result, IO};
use log::trace;
use log::{debug, trace};
use std::cell::RefCell;
use std::io::{Read, Seek, Write};
use std::rc::Rc;
@@ -8,6 +8,7 @@ pub struct GenericIO {}
impl GenericIO {
pub fn new() -> Result<Self> {
debug!("Using IO backend 'generic'");
Ok(Self {})
}
}

1
core/io/io_uring.rs
View File

@@ -70,6 +70,7 @@ impl UringIO {
}; MAX_IOVECS],
next_iovec: 0,
};
debug!("Using IO backend 'io-uring'");
Ok(Self {
inner: Rc::new(RefCell::new(inner)),
})

2
core/io/memory.rs
View File

@@ -1,6 +1,7 @@
use super::{Buffer, Completion, File, OpenFlags, IO};
use crate::Result;
use log::debug;
use std::{
cell::{RefCell, RefMut},
collections::BTreeMap,
@@ -20,6 +21,7 @@ type MemPage = Box<[u8; PAGE_SIZE]>;
impl MemoryIO {
#[allow(clippy::arc_with_non_send_sync)]
pub fn new() -> Result<Arc<Self>> {
debug!("Using IO backend 'memory'");
Ok(Arc::new(Self {
pages: RefCell::new(BTreeMap::new()),
size: RefCell::new(0),

4
core/io/mod.rs
View File

@@ -166,11 +166,15 @@ impl Buffer {
cfg_block! {
#[cfg(all(target_os = "linux", feature = "io_uring"))] {
mod io_uring;
pub use io_uring::UringIO;
mod unix;
pub use unix::UnixIO;
pub use io_uring::UringIO as PlatformIO;
}
#[cfg(any(all(target_os = "linux",not(feature = "io_uring")), target_os = "macos"))] {
mod unix;
pub use unix::UnixIO;
pub use unix::UnixIO as PlatformIO;
}

3
core/io/unix.rs
View File

@@ -4,7 +4,7 @@ use crate::Result;
use super::{Completion, File, OpenFlags, IO};
use libc::{c_short, fcntl, flock, F_SETLK};
use log::trace;
use log::{debug, trace};
use polling::{Event, Events, Poller};
use rustix::fd::{AsFd, AsRawFd};
use rustix::fs::OpenOptionsExt;
@@ -22,6 +22,7 @@ pub struct UnixIO {
impl UnixIO {
pub fn new() -> Result<Self> {
debug!("Using IO backend 'syscall'");
Ok(Self {
poller: Rc::new(RefCell::new(Poller::new()?)),
events: Rc::new(RefCell::new(Events::new())),

3
core/io/windows.rs
View File

@@ -1,5 +1,5 @@
use crate::{Completion, File, LimboError, OpenFlags, Result, IO};
use log::trace;
use log::{debug, trace};
use std::cell::RefCell;
use std::io::{Read, Seek, Write};
use std::rc::Rc;
@@ -8,6 +8,7 @@ pub struct WindowsIO {}
impl WindowsIO {
pub fn new() -> Result<Self> {
debug!("Using IO backend 'syscall'");
Ok(Self {})
}
}

5
core/lib.rs
View File

@@ -44,8 +44,11 @@ pub type Result<T> = std::result::Result<T, error::LimboError>;
use crate::translate::optimizer::optimize_plan;
pub use io::OpenFlags;
#[cfg(feature = "fs")]
pub use io::PlatformIO;
#[cfg(all(feature = "fs", target_family = "unix"))]
pub use io::UnixIO;
#[cfg(all(feature = "fs", target_os = "linux", feature = "io_uring"))]
pub use io::UringIO;
pub use io::{Buffer, Completion, File, MemoryIO, WriteCompletion, IO};
pub use storage::buffer_pool::BufferPool;
pub use storage::database::DatabaseStorage;