Merge 'Add libc fault injection to Antithesis' from Pekka Enberg

Fixes #2644

Reviewed-by: Preston Thorpe <preston@turso.tech>
Reviewed-by: Pere Diaz Bou <pere-altea@homail.com>

Closes #2647

CONTRIBUTING.md

@@ -211,6 +211,21 @@ Once Maturin is installed, you can build the crate and install it as a Python mo
 cd bindings/python && maturin develop
 ```
 
+## Fault injection with unreliable libc
+
+First, build the unreliable libc:
+
+```
+cd testing/unreliable-libc
+make
+```
+
+The run the stress testing tool with fault injection enabled:
+
+```
+RUST_BACKTRACE=1 LD_PRELOAD=./testing/unreliable-libc/unreliable-libc.so cargo run -p turso_stress -- --nr-iterations 10000
+```
+
 ## Antithesis
 
 Antithesis is a testing platform for finding bugs with reproducibility. In

Dockerfile.antithesis

@@ -31,6 +31,7 @@ COPY ./stress ./stress/
 COPY ./sync ./sync/
 COPY ./vendored ./vendored/
 COPY ./testing/sqlite_test_ext ./testing/sqlite_test_ext/
+COPY ./testing/unreliable-libc ./testing/unreliable-libc/
 RUN cargo chef prepare --bin turso_stress --recipe-path recipe.json
 
 #
@@ -73,6 +74,7 @@ COPY --from=planner /app/sync ./sync/
 COPY --from=planner /app/parser ./parser/
 COPY --from=planner /app/vendored ./vendored/
 COPY --from=planner /app/testing/sqlite_test_ext ./testing/sqlite_test_ext/
+COPY --from=planner /app/testing/unreliable-libc ./testing/unreliable-libc/
 
 RUN if [ "$antithesis" = "true" ]; then \
         cp /opt/antithesis/libvoidstar.so /usr/lib/libvoidstar.so && \
@@ -85,6 +87,9 @@ RUN if [ "$antithesis" = "true" ]; then \
 WORKDIR /app/bindings/python
 RUN maturin build
 
+WORKDIR /app/testing/unreliable-libc
+RUN make
+
 #
 # The final image.
 #
@@ -96,6 +101,7 @@ RUN pip install antithesis
 WORKDIR /app
 EXPOSE 8080
 COPY --from=builder /usr/lib/libvoidstar.so* /usr/lib/
+COPY --from=builder /app/testing/unreliable-libc/unreliable-libc.so /usr/lib/
 COPY --from=builder /app/target/antithesis/turso_stress /bin/turso_stress
 COPY --from=builder /app/target/antithesis/turso_stress /symbols
 COPY stress/docker-entrypoint.sh /bin
@@ -108,6 +114,7 @@ WORKDIR /app
 COPY ./antithesis-tests/bank-test/*.py /opt/antithesis/test/v1/bank-test/
 COPY ./antithesis-tests/stress-composer/*.py /opt/antithesis/test/v1/stress-composer/
 COPY ./antithesis-tests/stress /opt/antithesis/test/v1/stress
+COPY ./antithesis-tests/stress-unreliable /opt/antithesis/test/v1/stress-unreliable
 RUN chmod 777 -R /opt/antithesis/test/v1
 
 RUN mkdir /opt/antithesis/catalog

antithesis-tests/stress-unreliable/singleton_driver_stress.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+LD_PRELOAD=/usr/lib/unreliable-libc.so /bin/turso_stress --silent --nr-iterations 10000

core/io/unix.rs

@@ -192,19 +192,25 @@ impl File for UnixFile {
             .file
             .try_lock()
             .ok_or_else(|| LimboError::LockingError("Failed locking file".to_string()))?;
-        let result = {
+        let result = unsafe {
             let r = c.as_read();
             let buf = r.buf();
-            rustix::io::pread(file.as_fd(), buf.as_mut_slice(), pos as u64)
+            let slice = buf.as_mut_slice();
+            libc::pread(
+                file.as_raw_fd(),
+                slice.as_mut_ptr() as *mut libc::c_void,
+                slice.len(),
+                pos as libc::off_t,
+            )
         };
-        match result {
-            Ok(n) => {
-                trace!("pread n: {}", n);
-                // Read succeeded immediately
-                c.complete(n as i32);
-                Ok(c)
-            }
-            Err(e) => Err(e.into()),
+        if result == -1 {
+            let e = std::io::Error::last_os_error();
+            Err(e.into())
+        } else {
+            trace!("pread n: {}", result);
+            // Read succeeded immediately
+            c.complete(result as i32);
+            Ok(c)
         }
     }
 
@@ -260,14 +266,14 @@ impl File for UnixFile {
             .file
             .try_lock()
             .ok_or_else(|| LimboError::LockingError("Failed locking file".to_string()))?;
-        let result = fs::fsync(file.as_fd());
-        match result {
-            Ok(()) => {
-                trace!("fsync");
-                c.complete(0);
-                Ok(c)
-            }
-            Err(e) => Err(e.into()),
+        let result = unsafe { libc::fsync(file.as_raw_fd()) };
+        if result == -1 {
+            let e = std::io::Error::last_os_error();
+            Err(e.into())
+        } else {
+            trace!("fsync");
+            c.complete(0);
+            Ok(c)
         }
     }
 

stress/main.rs

@@ -496,7 +496,11 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
                             println!("Error creating table: {e}");
                         }
                     }
-                    _ => panic!("Error creating table: {}", e),
+                    _ => {
+                        println!("Error creating table: {e}");
+                        // Exit on any other error during table creation
+                        std::process::exit(1);
+                    }
                 }
             }
         }

testing/unreliable-libc/Makefile

@@ -0,0 +1,25 @@
+CC = gcc
+CFLAGS = -Wall -Wextra -fPIC
+LDFLAGS = -shared
+TARGET = unreliable-libc.so
+SRCS = file.c
+OBJS = $(SRCS:.c=.o)
+
+.PHONY: all clean
+
+all: $(TARGET)
+
+$(TARGET): $(OBJS)
+	$(CC) $(LDFLAGS) -o $@ $^ -ldl
+
+%.o: %.c
+	$(CC) $(CFLAGS) -c $< -o $@
+
+clean:
+	rm -f $(OBJS) $(TARGET)
+
+install: $(TARGET)
+	install -m 755 $(TARGET) /usr/local/lib
+
+uninstall:
+	rm -f /usr/local/lib/$(TARGET)

testing/unreliable-libc/file.c

@@ -0,0 +1,99 @@
+#define _GNU_SOURCE
+#include <sys/uio.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <unistd.h>
+#include <dlfcn.h>
+#include <errno.h>
+#include <stdio.h>
+#include <time.h>
+
+static double probabilities[] = {
+	[ENOSPC] = 0.01,
+	[EIO] = 0.01,
+};
+
+static bool chance(double probability)
+{
+	double event = drand48();
+	return event < probability;
+}
+
+static bool inject_fault(int error)
+{
+	double probability = probabilities[error];
+	if (chance(probability)) {
+		errno = error;
+		return true;
+	}
+	errno = 0;
+	return false;
+}
+
+static ssize_t (*libc_pwrite) (int, const void *, size_t, off_t);
+
+ssize_t pwrite(int fd, const void *buf, size_t count, off_t offset)
+{
+	if (libc_pwrite == NULL) {
+		libc_pwrite = dlsym(RTLD_NEXT, "pwrite");
+	}
+	if (inject_fault(ENOSPC)) {
+		printf("%s: injecting fault NOSPC\n", __func__);
+		return -1;
+	}
+	if (inject_fault(EIO)) {
+		printf("%s: injecting fault EIO\n", __func__);
+		return -1;
+	}
+	return libc_pwrite(fd, buf, count, offset);
+}
+
+static ssize_t (*libc_pwritev) (int, const struct iovec *, int, off_t);
+
+ssize_t pwritev(int fd, const struct iovec *iov, int iovcnt, off_t offset)
+{
+	if (libc_pwritev == NULL) {
+		libc_pwritev = dlsym(RTLD_NEXT, "pwritev");
+	}
+	if (inject_fault(ENOSPC)) {
+		printf("%s: injecting fault NOSPC\n", __func__);
+		return -1;
+	}
+	if (inject_fault(EIO)) {
+		printf("%s: injecting fault EIO\n", __func__);
+		return -1;
+	}
+	return libc_pwritev(fd, iov, iovcnt, offset);
+}
+
+static int (*libc_fsync) (int);
+
+int fsync(int fd)
+{
+	if (libc_fsync == NULL) {
+		libc_fsync = dlsym(RTLD_NEXT, "fsync");
+	}
+	if (inject_fault(ENOSPC)) {
+		printf("%s: injecting fault NOSPC\n", __func__);
+		return -1;
+	}
+	if (inject_fault(EIO)) {
+		printf("%s: injecting fault EIO\n", __func__);
+		return -1;
+	}
+	return libc_fsync(fd);
+}
+
+__attribute__((constructor))
+static void init(void)
+{
+	char *env_seed = getenv("UNRELIABLE_LIBC_SEED");
+	long seedval;
+	if (!env_seed) {
+		seedval = time(NULL);
+	} else {
+		seedval = atoi(env_seed);
+	}
+	printf("UNRELIABLE_LIBC_SEED = %ld\n", seedval);
+	srand48(seedval);
+}