From 9233f48e08196f00c2bf2eda5cdd69916ffca81f Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Wed, 20 Aug 2025 11:34:18 +0300 Subject: [PATCH 1/8] core/io: Switch Unix I/O operations to use libc We need it for LD_PRELOAD fault injection to work. --- core/io/unix.rs | 42 ++++++++++++++++++++++++------------------ 1 file changed, 24 insertions(+), 18 deletions(-) diff --git a/core/io/unix.rs b/core/io/unix.rs index e63456205..0fe95aa4b 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -192,19 +192,25 @@ impl File for UnixFile { .file .try_lock() .ok_or_else(|| LimboError::LockingError("Failed locking file".to_string()))?; - let result = { + let result = unsafe { let r = c.as_read(); let buf = r.buf(); - rustix::io::pread(file.as_fd(), buf.as_mut_slice(), pos as u64) + let slice = buf.as_mut_slice(); + libc::pread( + file.as_raw_fd(), + slice.as_mut_ptr() as *mut libc::c_void, + slice.len(), + pos as libc::off_t, + ) }; - match result { - Ok(n) => { - trace!("pread n: {}", n); - // Read succeeded immediately - c.complete(n as i32); - Ok(c) - } - Err(e) => Err(e.into()), + if result == -1 { + let e = std::io::Error::last_os_error(); + Err(e.into()) + } else { + trace!("pread n: {}", result); + // Read succeeded immediately + c.complete(result as i32); + Ok(c) } } @@ -260,14 +266,14 @@ impl File for UnixFile { .file .try_lock() .ok_or_else(|| LimboError::LockingError("Failed locking file".to_string()))?; - let result = fs::fsync(file.as_fd()); - match result { - Ok(()) => { - trace!("fsync"); - c.complete(0); - Ok(c) - } - Err(e) => Err(e.into()), + let result = unsafe { libc::fsync(file.as_raw_fd()) }; + if result == -1 { + let e = std::io::Error::last_os_error(); + Err(e.into()) + } else { + trace!("fsync"); + c.complete(0); + Ok(c) } } From 19456147ec8e0ea65f7bc9c11ac20e78ff43c331 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 18 Aug 2025 15:49:59 +0300 Subject: [PATCH 2/8] testing: Add unreliable libc --- testing/unreliable-libc/Makefile | 25 +++++++++ testing/unreliable-libc/file.c | 91 ++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100644 testing/unreliable-libc/Makefile create mode 100644 testing/unreliable-libc/file.c diff --git a/testing/unreliable-libc/Makefile b/testing/unreliable-libc/Makefile new file mode 100644 index 000000000..4b1a9eb57 --- /dev/null +++ b/testing/unreliable-libc/Makefile @@ -0,0 +1,25 @@ +CC = gcc +CFLAGS = -Wall -Wextra -fPIC +LDFLAGS = -shared +TARGET = unreliable-libc.so +SRCS = file.c +OBJS = $(SRCS:.c=.o) + +.PHONY: all clean + +all: $(TARGET) + +$(TARGET): $(OBJS) + $(CC) $(LDFLAGS) -o $@ $^ -ldl + +%.o: %.c + $(CC) $(CFLAGS) -c $< -o $@ + +clean: + rm -f $(OBJS) $(TARGET) + +install: $(TARGET) + install -m 755 $(TARGET) /usr/local/lib + +uninstall: + rm -f /usr/local/lib/$(TARGET) diff --git a/testing/unreliable-libc/file.c b/testing/unreliable-libc/file.c new file mode 100644 index 000000000..88f09ad90 --- /dev/null +++ b/testing/unreliable-libc/file.c @@ -0,0 +1,91 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static double probabilities[] = { + [ENOSPC] = 0.01, + [EIO] = 0.01, +}; + +static bool chance(double probability) +{ + double event = drand48(); + return event < probability; +} + +static bool inject_fault(int error) +{ + double probability = probabilities[error]; + if (chance(probability)) { + errno = error; + return true; + } + errno = 0; + return false; +} + +static ssize_t (*libc_pwrite) (int, const void *, size_t, off_t); + +ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) +{ + if (libc_pwrite == NULL) { + libc_pwrite = dlsym(RTLD_NEXT, "pwrite"); + } + if (inject_fault(ENOSPC)) { + printf("%s: injecting fault NOSPC\n", __func__); + return -1; + } + if (inject_fault(EIO)) { + printf("%s: injecting fault EIO\n", __func__); + return -1; + } + return libc_pwrite(fd, buf, count, offset); +} + +static ssize_t (*libc_pwritev) (int, const struct iovec *, int, off_t); + +ssize_t pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset) +{ + if (libc_pwritev == NULL) { + libc_pwritev = dlsym(RTLD_NEXT, "pwritev"); + } + if (inject_fault(ENOSPC)) { + printf("%s: injecting fault NOSPC\n", __func__); + return -1; + } + if (inject_fault(EIO)) { + printf("%s: injecting fault EIO\n", __func__); + return -1; + } + return libc_pwritev(fd, iov, iovcnt, offset); +} + +static int (*libc_fsync) (int); + +int fsync(int fd) +{ + if (libc_fsync == NULL) { + libc_fsync = dlsym(RTLD_NEXT, "fsync"); + } + if (inject_fault(ENOSPC)) { + printf("%s: injecting fault NOSPC\n", __func__); + return -1; + } + if (inject_fault(EIO)) { + printf("%s: injecting fault EIO\n", __func__); + return -1; + } + return libc_fsync(fd); +} + +__attribute__((constructor)) +static void init(void) +{ + srand48(time(NULL)); +} From 46eb3e27612cedddeca8c4701d31697636be5c46 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 19 Aug 2025 09:14:34 +0300 Subject: [PATCH 3/8] stress: Don't hang if table creation fails --- stress/main.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/stress/main.rs b/stress/main.rs index 05e35c589..2a71de13d 100644 --- a/stress/main.rs +++ b/stress/main.rs @@ -496,7 +496,11 @@ async fn main() -> Result<(), Box> { println!("Error creating table: {e}"); } } - _ => panic!("Error creating table: {}", e), + _ => { + println!("Error creating table: {e}"); + // Exit on any other error during table creation + std::process::exit(1); + } } } } From 39dd86623e941da4a27d3c95b3565fef87701021 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 18 Aug 2025 15:52:22 +0300 Subject: [PATCH 4/8] antithesis: Add unreliable libc to Docker image --- Dockerfile.antithesis | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Dockerfile.antithesis b/Dockerfile.antithesis index a59626790..f4acf8970 100644 --- a/Dockerfile.antithesis +++ b/Dockerfile.antithesis @@ -31,6 +31,7 @@ COPY ./stress ./stress/ COPY ./sync ./sync/ COPY ./vendored ./vendored/ COPY ./testing/sqlite_test_ext ./testing/sqlite_test_ext/ +COPY ./testing/unreliable-libc ./testing/unreliable-libc/ RUN cargo chef prepare --bin turso_stress --recipe-path recipe.json # @@ -73,6 +74,7 @@ COPY --from=planner /app/sync ./sync/ COPY --from=planner /app/parser ./parser/ COPY --from=planner /app/vendored ./vendored/ COPY --from=planner /app/testing/sqlite_test_ext ./testing/sqlite_test_ext/ +COPY --from=planner /app/testing/unreliable-libc ./testing/unreliable-libc/ RUN if [ "$antithesis" = "true" ]; then \ cp /opt/antithesis/libvoidstar.so /usr/lib/libvoidstar.so && \ @@ -85,6 +87,9 @@ RUN if [ "$antithesis" = "true" ]; then \ WORKDIR /app/bindings/python RUN maturin build +WORKDIR /app/testing/unreliable-libc +RUN make + # # The final image. # @@ -96,6 +101,7 @@ RUN pip install antithesis WORKDIR /app EXPOSE 8080 COPY --from=builder /usr/lib/libvoidstar.so* /usr/lib/ +COPY --from=builder /app/testing/unreliable-libc/unreliable-libc.so /usr/lib/ COPY --from=builder /app/target/antithesis/turso_stress /bin/turso_stress COPY --from=builder /app/target/antithesis/turso_stress /symbols COPY stress/docker-entrypoint.sh /bin From 551a8f92b1d5957621f9e33360ee6cf2047f57f0 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 18 Aug 2025 15:52:57 +0300 Subject: [PATCH 5/8] antithesis: Add unreliable libc stress template --- antithesis-tests/stress-unreliable/singleton_driver_stress.sh | 3 +++ 1 file changed, 3 insertions(+) create mode 100755 antithesis-tests/stress-unreliable/singleton_driver_stress.sh diff --git a/antithesis-tests/stress-unreliable/singleton_driver_stress.sh b/antithesis-tests/stress-unreliable/singleton_driver_stress.sh new file mode 100755 index 000000000..d36ad45ca --- /dev/null +++ b/antithesis-tests/stress-unreliable/singleton_driver_stress.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +LD_PRELOAD=/usr/lib/unreliable-libc.so /bin/turso_stress --silent --nr-iterations 10000 From 89d152a060ccd2e3a31c817e7113436194a93959 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Mon, 18 Aug 2025 15:53:59 +0300 Subject: [PATCH 6/8] antithesis: Add unreliable stress template to Docker image --- Dockerfile.antithesis | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.antithesis b/Dockerfile.antithesis index f4acf8970..d68af8721 100644 --- a/Dockerfile.antithesis +++ b/Dockerfile.antithesis @@ -114,6 +114,7 @@ WORKDIR /app COPY ./antithesis-tests/bank-test/*.py /opt/antithesis/test/v1/bank-test/ COPY ./antithesis-tests/stress-composer/*.py /opt/antithesis/test/v1/stress-composer/ COPY ./antithesis-tests/stress /opt/antithesis/test/v1/stress +COPY ./antithesis-tests/stress-unreliable /opt/antithesis/test/v1/stress-unreliable RUN chmod 777 -R /opt/antithesis/test/v1 RUN mkdir /opt/antithesis/catalog From 0a6c54100f7ec1958cfac3144fb645c841c09084 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 19 Aug 2025 09:15:49 +0300 Subject: [PATCH 7/8] Add fault injection steps to CONTRIBUTING.md --- CONTRIBUTING.md | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c4b4d007..b9bcf3afb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -211,6 +211,21 @@ Once Maturin is installed, you can build the crate and install it as a Python mo cd bindings/python && maturin develop ``` +## Fault injection with unreliable libc + +First, build the unreliable libc: + +``` +cd testing/unreliable-libc +make +``` + +The run the stress testing tool with fault injection enabled: + +``` +RUST_BACKTRACE=1 LD_PRELOAD=./testing/unreliable-libc/unreliable-libc.so cargo run -p turso_stress -- --nr-iterations 10000 +``` + ## Antithesis Antithesis is a testing platform for finding bugs with reproducibility. In From 7270e665307df8e62a2f4f63a3225d40e36474b0 Mon Sep 17 00:00:00 2001 From: Pekka Enberg Date: Tue, 19 Aug 2025 09:18:20 +0300 Subject: [PATCH 8/8] unreliable-libc: Make fault injection seed configurable --- testing/unreliable-libc/file.c | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/testing/unreliable-libc/file.c b/testing/unreliable-libc/file.c index 88f09ad90..9a580b513 100644 --- a/testing/unreliable-libc/file.c +++ b/testing/unreliable-libc/file.c @@ -87,5 +87,13 @@ int fsync(int fd) __attribute__((constructor)) static void init(void) { - srand48(time(NULL)); + char *env_seed = getenv("UNRELIABLE_LIBC_SEED"); + long seedval; + if (!env_seed) { + seedval = time(NULL); + } else { + seedval = atoi(env_seed); + } + printf("UNRELIABLE_LIBC_SEED = %ld\n", seedval); + srand48(seedval); }