diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 5c4b4d007..b9bcf3afb 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -211,6 +211,21 @@ Once Maturin is installed, you can build the crate and install it as a Python mo cd bindings/python && maturin develop ``` +## Fault injection with unreliable libc + +First, build the unreliable libc: + +``` +cd testing/unreliable-libc +make +``` + +The run the stress testing tool with fault injection enabled: + +``` +RUST_BACKTRACE=1 LD_PRELOAD=./testing/unreliable-libc/unreliable-libc.so cargo run -p turso_stress -- --nr-iterations 10000 +``` + ## Antithesis Antithesis is a testing platform for finding bugs with reproducibility. In diff --git a/Dockerfile.antithesis b/Dockerfile.antithesis index a59626790..d68af8721 100644 --- a/Dockerfile.antithesis +++ b/Dockerfile.antithesis @@ -31,6 +31,7 @@ COPY ./stress ./stress/ COPY ./sync ./sync/ COPY ./vendored ./vendored/ COPY ./testing/sqlite_test_ext ./testing/sqlite_test_ext/ +COPY ./testing/unreliable-libc ./testing/unreliable-libc/ RUN cargo chef prepare --bin turso_stress --recipe-path recipe.json # @@ -73,6 +74,7 @@ COPY --from=planner /app/sync ./sync/ COPY --from=planner /app/parser ./parser/ COPY --from=planner /app/vendored ./vendored/ COPY --from=planner /app/testing/sqlite_test_ext ./testing/sqlite_test_ext/ +COPY --from=planner /app/testing/unreliable-libc ./testing/unreliable-libc/ RUN if [ "$antithesis" = "true" ]; then \ cp /opt/antithesis/libvoidstar.so /usr/lib/libvoidstar.so && \ @@ -85,6 +87,9 @@ RUN if [ "$antithesis" = "true" ]; then \ WORKDIR /app/bindings/python RUN maturin build +WORKDIR /app/testing/unreliable-libc +RUN make + # # The final image. # @@ -96,6 +101,7 @@ RUN pip install antithesis WORKDIR /app EXPOSE 8080 COPY --from=builder /usr/lib/libvoidstar.so* /usr/lib/ +COPY --from=builder /app/testing/unreliable-libc/unreliable-libc.so /usr/lib/ COPY --from=builder /app/target/antithesis/turso_stress /bin/turso_stress COPY --from=builder /app/target/antithesis/turso_stress /symbols COPY stress/docker-entrypoint.sh /bin @@ -108,6 +114,7 @@ WORKDIR /app COPY ./antithesis-tests/bank-test/*.py /opt/antithesis/test/v1/bank-test/ COPY ./antithesis-tests/stress-composer/*.py /opt/antithesis/test/v1/stress-composer/ COPY ./antithesis-tests/stress /opt/antithesis/test/v1/stress +COPY ./antithesis-tests/stress-unreliable /opt/antithesis/test/v1/stress-unreliable RUN chmod 777 -R /opt/antithesis/test/v1 RUN mkdir /opt/antithesis/catalog diff --git a/antithesis-tests/stress-unreliable/singleton_driver_stress.sh b/antithesis-tests/stress-unreliable/singleton_driver_stress.sh new file mode 100755 index 000000000..d36ad45ca --- /dev/null +++ b/antithesis-tests/stress-unreliable/singleton_driver_stress.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +LD_PRELOAD=/usr/lib/unreliable-libc.so /bin/turso_stress --silent --nr-iterations 10000 diff --git a/core/io/unix.rs b/core/io/unix.rs index e63456205..0fe95aa4b 100644 --- a/core/io/unix.rs +++ b/core/io/unix.rs @@ -192,19 +192,25 @@ impl File for UnixFile { .file .try_lock() .ok_or_else(|| LimboError::LockingError("Failed locking file".to_string()))?; - let result = { + let result = unsafe { let r = c.as_read(); let buf = r.buf(); - rustix::io::pread(file.as_fd(), buf.as_mut_slice(), pos as u64) + let slice = buf.as_mut_slice(); + libc::pread( + file.as_raw_fd(), + slice.as_mut_ptr() as *mut libc::c_void, + slice.len(), + pos as libc::off_t, + ) }; - match result { - Ok(n) => { - trace!("pread n: {}", n); - // Read succeeded immediately - c.complete(n as i32); - Ok(c) - } - Err(e) => Err(e.into()), + if result == -1 { + let e = std::io::Error::last_os_error(); + Err(e.into()) + } else { + trace!("pread n: {}", result); + // Read succeeded immediately + c.complete(result as i32); + Ok(c) } } @@ -260,14 +266,14 @@ impl File for UnixFile { .file .try_lock() .ok_or_else(|| LimboError::LockingError("Failed locking file".to_string()))?; - let result = fs::fsync(file.as_fd()); - match result { - Ok(()) => { - trace!("fsync"); - c.complete(0); - Ok(c) - } - Err(e) => Err(e.into()), + let result = unsafe { libc::fsync(file.as_raw_fd()) }; + if result == -1 { + let e = std::io::Error::last_os_error(); + Err(e.into()) + } else { + trace!("fsync"); + c.complete(0); + Ok(c) } } diff --git a/stress/main.rs b/stress/main.rs index 05e35c589..2a71de13d 100644 --- a/stress/main.rs +++ b/stress/main.rs @@ -496,7 +496,11 @@ async fn main() -> Result<(), Box> { println!("Error creating table: {e}"); } } - _ => panic!("Error creating table: {}", e), + _ => { + println!("Error creating table: {e}"); + // Exit on any other error during table creation + std::process::exit(1); + } } } } diff --git a/testing/unreliable-libc/Makefile b/testing/unreliable-libc/Makefile new file mode 100644 index 000000000..4b1a9eb57 --- /dev/null +++ b/testing/unreliable-libc/Makefile @@ -0,0 +1,25 @@ +CC = gcc +CFLAGS = -Wall -Wextra -fPIC +LDFLAGS = -shared +TARGET = unreliable-libc.so +SRCS = file.c +OBJS = $(SRCS:.c=.o) + +.PHONY: all clean + +all: $(TARGET) + +$(TARGET): $(OBJS) + $(CC) $(LDFLAGS) -o $@ $^ -ldl + +%.o: %.c + $(CC) $(CFLAGS) -c $< -o $@ + +clean: + rm -f $(OBJS) $(TARGET) + +install: $(TARGET) + install -m 755 $(TARGET) /usr/local/lib + +uninstall: + rm -f /usr/local/lib/$(TARGET) diff --git a/testing/unreliable-libc/file.c b/testing/unreliable-libc/file.c new file mode 100644 index 000000000..9a580b513 --- /dev/null +++ b/testing/unreliable-libc/file.c @@ -0,0 +1,99 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include + +static double probabilities[] = { + [ENOSPC] = 0.01, + [EIO] = 0.01, +}; + +static bool chance(double probability) +{ + double event = drand48(); + return event < probability; +} + +static bool inject_fault(int error) +{ + double probability = probabilities[error]; + if (chance(probability)) { + errno = error; + return true; + } + errno = 0; + return false; +} + +static ssize_t (*libc_pwrite) (int, const void *, size_t, off_t); + +ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset) +{ + if (libc_pwrite == NULL) { + libc_pwrite = dlsym(RTLD_NEXT, "pwrite"); + } + if (inject_fault(ENOSPC)) { + printf("%s: injecting fault NOSPC\n", __func__); + return -1; + } + if (inject_fault(EIO)) { + printf("%s: injecting fault EIO\n", __func__); + return -1; + } + return libc_pwrite(fd, buf, count, offset); +} + +static ssize_t (*libc_pwritev) (int, const struct iovec *, int, off_t); + +ssize_t pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset) +{ + if (libc_pwritev == NULL) { + libc_pwritev = dlsym(RTLD_NEXT, "pwritev"); + } + if (inject_fault(ENOSPC)) { + printf("%s: injecting fault NOSPC\n", __func__); + return -1; + } + if (inject_fault(EIO)) { + printf("%s: injecting fault EIO\n", __func__); + return -1; + } + return libc_pwritev(fd, iov, iovcnt, offset); +} + +static int (*libc_fsync) (int); + +int fsync(int fd) +{ + if (libc_fsync == NULL) { + libc_fsync = dlsym(RTLD_NEXT, "fsync"); + } + if (inject_fault(ENOSPC)) { + printf("%s: injecting fault NOSPC\n", __func__); + return -1; + } + if (inject_fault(EIO)) { + printf("%s: injecting fault EIO\n", __func__); + return -1; + } + return libc_fsync(fd); +} + +__attribute__((constructor)) +static void init(void) +{ + char *env_seed = getenv("UNRELIABLE_LIBC_SEED"); + long seedval; + if (!env_seed) { + seedval = time(NULL); + } else { + seedval = atoi(env_seed); + } + printf("UNRELIABLE_LIBC_SEED = %ld\n", seedval); + srand48(seedval); +}