aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2026-01-21 09:04:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'Rename IO backends to support Linux platforms without io_uring' from Jorge López Tello

Browse Source 
As was suggested in #502 I have renamed the IO backends from `darwin` to
`unix` and from `linux` to `io_uring`. The `unix` backend is now
available to Linux platforms without io_uring.

On the topic of letting platforms disable io_uring support, the primary
example is Google, which posted this
[blog](https://security.googleblog.com/2023/06/learnings-from-kctf-
vrps-42-linux.html) in 2023 about disabling io_uring by default in most
of their servers/platforms, including Android (prime candidate user for
a SQLite rewrite).

Closes #628
This commit is contained in:
Pekka Enberg
2025-01-07 20:33:03 +02:00
parent fbb5ddd8f1 511c0b495d
commit 3a853ada57
5 changed files with 43 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
core/Cargo.toml
View File

@@ -14,7 +14,7 @@ name = "limbo_core"
path = "lib.rs"
[features]
default = ["fs", "json", "uuid"]
default = ["fs", "json", "uuid", "io_uring"]
fs = []
json = [
"dep:jsonb",
@@ -22,11 +22,12 @@ json = [
"dep:pest_derive",
]
uuid = ["dep:uuid"]
io_uring = ["dep:io-uring"]
[target.'cfg(target_os = "linux")'.dependencies]
io-uring = "0.6.1"
io-uring = { version = "0.6.1", optional = true }
[target.'cfg(target_os = "macos")'.dependencies]
[target.'cfg(target_family = "unix")'.dependencies]
polling = "3.7.2"
rustix = "0.38.34"

6
core/error.rs
View File

@@ -19,12 +19,12 @@ pub enum LimboError {
EnvVarError(#[from] std::env::VarError),
#[error("I/O error: {0}")]
IOError(#[from] std::io::Error),
#[cfg(target_os = "linux")]
#[cfg(all(target_os = "linux", feature = "io_uring"))]
#[error("I/O error: {0}")]
LinuxIOError(String),
UringIOError(String),
#[error("Locking error: {0}")]
LockingError(String),
#[cfg(target_os = "macos")]
#[cfg(target_family = "unix")]
#[error("I/O error: {0}")]
RustixIOError(#[from] rustix::io::Errno),
#[error("Parse error: {0}")]

40
core/io/linux.rs → core/io/io_uring.rs
View File

@@ -14,15 +14,15 @@ const MAX_IOVECS: usize = 128;
const SQPOLL_IDLE: u32 = 1000;
#[derive(Debug, Error)]
enum LinuxIOError {
enum UringIOError {
IOUringCQError(i32),
}
// Implement the Display trait to customize error messages
impl fmt::Display for LinuxIOError {
impl fmt::Display for UringIOError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
LinuxIOError::IOUringCQError(code) => write!(
UringIOError::IOUringCQError(code) => write!(
f,
"IOUring completion queue error occurred with code {}",
code
@@ -31,8 +31,8 @@ impl fmt::Display for LinuxIOError {
}
}
pub struct LinuxIO {
inner: Rc<RefCell<InnerLinuxIO>>,
pub struct UringIO {
inner: Rc<RefCell<InnerUringIO>>,
}
struct WrappedIOUring {
@@ -42,13 +42,13 @@ struct WrappedIOUring {
key: u64,
}
struct InnerLinuxIO {
struct InnerUringIO {
ring: WrappedIOUring,
iovecs: [iovec; MAX_IOVECS],
next_iovec: usize,
}
impl LinuxIO {
impl UringIO {
pub fn new() -> Result<Self> {
let ring = match io_uring::IoUring::builder()
.setup_sqpoll(SQPOLL_IDLE)
@@ -57,7 +57,7 @@ impl LinuxIO {
Ok(ring) => ring,
Err(_) => io_uring::IoUring::new(MAX_IOVECS as u32)?,
};
let inner = InnerLinuxIO {
let inner = InnerUringIO {
ring: WrappedIOUring {
ring,
pending_ops: 0,
@@ -76,7 +76,7 @@ impl LinuxIO {
}
}
impl InnerLinuxIO {
impl InnerUringIO {
pub fn get_iovec(&mut self, buf: *const u8, len: usize) -> &iovec {
let iovec = &mut self.iovecs[self.next_iovec];
iovec.iov_base = buf as *mut std::ffi::c_void;
@@ -125,7 +125,7 @@ impl WrappedIOUring {
}
}
impl IO for LinuxIO {
impl IO for UringIO {
fn open_file(&self, path: &str, flags: OpenFlags, direct: bool) -> Result<Rc<dyn File>> {
trace!("open_file(path = {})", path);
let file = std::fs::File::options()
@@ -142,14 +142,14 @@ impl IO for LinuxIO {
Err(error) => debug!("Error {error:?} returned when setting O_DIRECT flag to read file. The performance of the system may be affected"),
};
}
let linux_file = Rc::new(LinuxFile {
let uring_file = Rc::new(UringFile {
io: self.inner.clone(),
file,
});
if std::env::var(common::ENV_DISABLE_FILE_LOCK).is_err() {
linux_file.lock_file(true)?;
uring_file.lock_file(true)?;
}
Ok(linux_file)
Ok(uring_file)
}
fn run_once(&self) -> Result<()> {
@@ -165,9 +165,9 @@ impl IO for LinuxIO {
while let Some(cqe) = ring.get_completion() {
let result = cqe.result();
if result < 0 {
return Err(LimboError::LinuxIOError(format!(
return Err(LimboError::UringIOError(format!(
"{} cqe: {:?}",
LinuxIOError::IOUringCQError(result),
UringIOError::IOUringCQError(result),
cqe
)));
}
@@ -191,12 +191,12 @@ impl IO for LinuxIO {
}
}
pub struct LinuxFile {
io: Rc<RefCell<InnerLinuxIO>>,
pub struct UringFile {
io: Rc<RefCell<InnerUringIO>>,
file: std::fs::File,
}
impl File for LinuxFile {
impl File for UringFile {
fn lock_file(&self, exclusive: bool) -> Result<()> {
let fd = self.file.as_raw_fd();
let flock = flock {
@@ -306,7 +306,7 @@ impl File for LinuxFile {
}
}
impl Drop for LinuxFile {
impl Drop for UringFile {
fn drop(&mut self) {
self.unlock_file().expect("Failed to unlock file");
}
@@ -319,6 +319,6 @@ mod tests {
#[test]
fn test_multiple_processes_cannot_open_file() {
common::tests::test_multiple_processes_cannot_open_file(LinuxIO::new);
common::tests::test_multiple_processes_cannot_open_file(UringIO::new);
}
}

12
core/io/mod.rs
View File

@@ -164,14 +164,14 @@ impl Buffer {
}
cfg_block! {
#[cfg(target_os = "linux")] {
mod linux;
pub use linux::LinuxIO as PlatformIO;
#[cfg(all(target_os = "linux", feature = "io_uring"))] {
mod io_uring;
pub use io_uring::UringIO as PlatformIO;
}
#[cfg(target_os = "macos")] {
mod darwin;
pub use darwin::DarwinIO as PlatformIO;
#[cfg(any(all(target_os = "linux",not(feature = "io_uring")), target_os = "macos"))] {
mod unix;
pub use unix::UnixIO as PlatformIO;
}
#[cfg(target_os = "windows")] {

20
core/io/darwin.rs → core/io/unix.rs
View File

@@ -14,13 +14,13 @@ use std::collections::HashMap;
use std::io::{Read, Seek, Write};
use std::rc::Rc;
pub struct DarwinIO {
pub struct UnixIO {
poller: Rc<RefCell<Poller>>,
events: Rc<RefCell<Events>>,
callbacks: Rc<RefCell<HashMap<usize, CompletionCallback>>>,
}
impl DarwinIO {
impl UnixIO {
pub fn new() -> Result<Self> {
Ok(Self {
poller: Rc::new(RefCell::new(Poller::new()?)),
@@ -30,7 +30,7 @@ impl DarwinIO {
}
}
impl IO for DarwinIO {
impl IO for UnixIO {
fn open_file(&self, path: &str, flags: OpenFlags, _direct: bool) -> Result<Rc<dyn File>> {
trace!("open_file(path = {})", path);
let file = std::fs::File::options()
@@ -40,15 +40,15 @@ impl IO for DarwinIO {
.create(matches!(flags, OpenFlags::Create))
.open(path)?;
let darwin_file = Rc::new(DarwinFile {
let unix_file = Rc::new(UnixFile {
file: Rc::new(RefCell::new(file)),
poller: self.poller.clone(),
callbacks: self.callbacks.clone(),
});
if std::env::var(common::ENV_DISABLE_FILE_LOCK).is_err() {
darwin_file.lock_file(true)?;
unix_file.lock_file(true)?;
}
Ok(darwin_file)
Ok(unix_file)
}
fn run_once(&self) -> Result<()> {
@@ -127,13 +127,13 @@ enum CompletionCallback {
),
}
pub struct DarwinFile {
pub struct UnixFile {
file: Rc<RefCell<std::fs::File>>,
poller: Rc<RefCell<polling::Poller>>,
callbacks: Rc<RefCell<HashMap<usize, CompletionCallback>>>,
}
impl File for DarwinFile {
impl File for UnixFile {
fn lock_file(&self, exclusive: bool) -> Result<()> {
let fd = self.file.borrow().as_raw_fd();
let flock = flock {
@@ -279,7 +279,7 @@ impl File for DarwinFile {
}
}
impl Drop for DarwinFile {
impl Drop for UnixFile {
fn drop(&mut self) {
self.unlock_file().expect("Failed to unlock file");
}
@@ -291,6 +291,6 @@ mod tests {
#[test]
fn test_multiple_processes_cannot_open_file() {
common::tests::test_multiple_processes_cannot_open_file(DarwinIO::new);
common::tests::test_multiple_processes_cannot_open_file(UnixIO::new);
}
}