,
@@ -29,7 +28,6 @@ impl LimboConnection {
Box::into_raw(Box::new(self)) as jlong
}
- #[allow(dead_code)]
pub fn drop(ptr: jlong) {
let _boxed = unsafe { Box::from_raw(ptr as *mut LimboConnection) };
}
@@ -43,6 +41,15 @@ pub fn to_limbo_connection(ptr: jlong) -> Result<&'static mut LimboConnection> {
}
}
+#[no_mangle]
+pub extern "system" fn Java_org_github_tursodatabase_core_LimboConnection__1close<'local>(
+ _env: JNIEnv<'local>,
+ _obj: JObject<'local>,
+ connection_ptr: jlong,
+) {
+ LimboConnection::drop(connection_ptr);
+}
+
#[no_mangle]
pub extern "system" fn Java_org_github_tursodatabase_core_LimboConnection_prepareUtf8<'local>(
mut env: JNIEnv<'local>,
diff --git a/bindings/java/rs_src/limbo_statement.rs b/bindings/java/rs_src/limbo_statement.rs
index 7de4b2c19..aed8e7d99 100644
--- a/bindings/java/rs_src/limbo_statement.rs
+++ b/bindings/java/rs_src/limbo_statement.rs
@@ -29,7 +29,6 @@ impl LimboStatement {
Box::into_raw(Box::new(self)) as jlong
}
- #[allow(dead_code)]
pub fn drop(ptr: jlong) {
let _boxed = unsafe { Box::from_raw(ptr as *mut LimboStatement) };
}
@@ -88,6 +87,15 @@ pub extern "system" fn Java_org_github_tursodatabase_core_LimboStatement_step<'l
}
}
+#[no_mangle]
+pub extern "system" fn Java_org_github_tursodatabase_core_LimboStatement__1close<'local>(
+ _env: JNIEnv<'local>,
+ _obj: JObject<'local>,
+ stmt_ptr: jlong,
+) {
+ LimboStatement::drop(stmt_ptr);
+}
+
fn row_to_obj_array<'local>(
env: &mut JNIEnv<'local>,
row: &limbo_core::Row,
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java
index 63c6e57d7..6928c32cc 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/JDBC.java
@@ -3,20 +3,23 @@ package org.github.tursodatabase;
import java.sql.*;
import java.util.Locale;
import java.util.Properties;
-import java.util.logging.Logger;
import org.github.tursodatabase.annotations.Nullable;
import org.github.tursodatabase.annotations.SkipNullableCheck;
import org.github.tursodatabase.core.LimboConnection;
import org.github.tursodatabase.jdbc4.JDBC4Connection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
public class JDBC implements Driver {
+ private static final Logger logger = LoggerFactory.getLogger(JDBC.class);
+
private static final String VALID_URL_PREFIX = "jdbc:sqlite:";
static {
try {
DriverManager.registerDriver(new JDBC());
} catch (Exception e) {
- // TODO: log
+ logger.error("Failed to register driver", e);
}
}
@@ -72,7 +75,7 @@ public class JDBC implements Driver {
@Override
@SkipNullableCheck
- public Logger getParentLogger() throws SQLFeatureNotSupportedException {
+ public java.util.logging.Logger getParentLogger() throws SQLFeatureNotSupportedException {
// TODO
return null;
}
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java
index cd200f74f..34e5692e3 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboConnection.java
@@ -16,6 +16,7 @@ public abstract class LimboConnection implements Connection {
private final long connectionPtr;
private final AbstractDB database;
+ private boolean closed;
public LimboConnection(String url, String filePath) throws SQLException {
this(url, filePath, new Properties());
@@ -28,24 +29,8 @@ public abstract class LimboConnection implements Connection {
* @param filePath path to file
*/
public LimboConnection(String url, String filePath, Properties properties) throws SQLException {
- AbstractDB db = null;
-
- try {
- db = open(url, filePath, properties);
- } catch (Throwable t) {
- try {
- if (db != null) {
- db.close();
- }
- } catch (Throwable t2) {
- t.addSuppressed(t2);
- }
-
- throw t;
- }
-
- this.database = db;
- this.connectionPtr = db.connect();
+ this.database = open(url, filePath, properties);
+ this.connectionPtr = this.database.connect();
}
private static AbstractDB open(String url, String filePath, Properties properties)
@@ -59,13 +44,18 @@ public abstract class LimboConnection implements Connection {
@Override
public void close() throws SQLException {
- if (isClosed()) return;
- database.close();
+ if (isClosed()) {
+ return;
+ }
+ this._close(this.connectionPtr);
+ this.closed = true;
}
+ private native void _close(long connectionPtr);
+
@Override
public boolean isClosed() throws SQLException {
- return database.isClosed();
+ return closed;
}
public AbstractDB getDatabase() {
@@ -114,12 +104,15 @@ public abstract class LimboConnection implements Connection {
*/
protected void checkCursor(int resultSetType, int resultSetConcurrency, int resultSetHoldability)
throws SQLException {
- if (resultSetType != ResultSet.TYPE_FORWARD_ONLY)
+ if (resultSetType != ResultSet.TYPE_FORWARD_ONLY) {
throw new SQLException("SQLite only supports TYPE_FORWARD_ONLY cursors");
- if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY)
+ }
+ if (resultSetConcurrency != ResultSet.CONCUR_READ_ONLY) {
throw new SQLException("SQLite only supports CONCUR_READ_ONLY cursors");
- if (resultSetHoldability != ResultSet.CLOSE_CURSORS_AT_COMMIT)
+ }
+ if (resultSetHoldability != ResultSet.CLOSE_CURSORS_AT_COMMIT) {
throw new SQLException("SQLite only supports closing cursors at commit");
+ }
}
public void setBusyTimeout(int busyTimeout) {
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java
index c32eaadf9..ac3f03b5c 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboDB.java
@@ -2,6 +2,10 @@ package org.github.tursodatabase.core;
import static org.github.tursodatabase.utils.ByteArrayUtils.stringToUtf8ByteArray;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
import java.sql.SQLException;
import java.util.concurrent.locks.ReentrantLock;
import org.github.tursodatabase.LimboErrorCode;
@@ -30,17 +34,136 @@ public final class LimboDB extends AbstractDB {
}
}
- /** Loads the SQLite interface backend. */
+ /**
+ * Enum representing different architectures and their corresponding library paths and file
+ * extensions.
+ */
+ enum Architecture {
+ MACOS_ARM64("libs/macos_arm64/lib_limbo_java.dylib", ".dylib"),
+ MACOS_X86("libs/macos_x86/lib_limbo_java.dylib", ".dylib"),
+ WINDOWS("libs/windows/lib_limbo_java.dll", ".dll"),
+ UNSUPPORTED("", "");
+
+ private final String libPath;
+ private final String fileExtension;
+
+ Architecture(String libPath, String fileExtension) {
+ this.libPath = libPath;
+ this.fileExtension = fileExtension;
+ }
+
+ public String getLibPath() {
+ return libPath;
+ }
+
+ public String getFileExtension() {
+ return fileExtension;
+ }
+
+ public static Architecture detect() {
+ String osName = System.getProperty("os.name").toLowerCase();
+ String osArch = System.getProperty("os.arch").toLowerCase();
+
+ if (osName.contains("mac")) {
+ if (osArch.contains("aarch64") || osArch.contains("arm64")) {
+ return MACOS_ARM64;
+ } else if (osArch.contains("x86_64") || osArch.contains("amd64")) {
+ return MACOS_X86;
+ }
+ } else if (osName.contains("win")) {
+ return WINDOWS;
+ }
+
+ return UNSUPPORTED;
+ }
+ }
+
+ /**
+ * This method attempts to load the native library required for Limbo operations. It first tries
+ * to load the library from the system's library path using {@link #loadFromSystemPath()}. If that
+ * fails, it attempts to load the library from the JAR file using {@link #loadFromJar()}. If
+ * either method succeeds, the `isLoaded` flag is set to true. If both methods fail, an {@link
+ * InternalError} is thrown indicating that the necessary native library could not be loaded.
+ *
+ * @throws InternalError if the native library cannot be loaded from either the system path or the
+ * JAR file.
+ */
public static void load() {
if (isLoaded) {
return;
}
+ if (loadFromSystemPath() || loadFromJar()) {
+ isLoaded = true;
+ return;
+ }
+
+ throw new InternalError("Unable to load necessary native library");
+ }
+
+ /**
+ * Load the native library from the system path.
+ *
+ * This method attempts to load the native library named "_limbo_java" from the system's
+ * library path. If the library is successfully loaded, the `isLoaded` flag is set to true.
+ *
+ * @return true if the library was successfully loaded, false otherwise.
+ */
+ private static boolean loadFromSystemPath() {
try {
System.loadLibrary("_limbo_java");
- } finally {
- isLoaded = true;
+ return true;
+ } catch (Throwable t) {
+ logger.info("Unable to load from default path: {}", String.valueOf(t));
}
+
+ return false;
+ }
+
+ /**
+ * Load the native library from the JAR file.
+ *
+ *
By default, native libraries are packaged within the JAR file. This method extracts the
+ * appropriate native library for the current operating system and architecture from the JAR and
+ * loads it.
+ *
+ * @return true if the library was successfully loaded, false otherwise.
+ */
+ private static boolean loadFromJar() {
+ Architecture arch = Architecture.detect();
+ if (arch == Architecture.UNSUPPORTED) {
+ logger.info("Unsupported OS or architecture");
+ return false;
+ }
+
+ try {
+ InputStream is = LimboDB.class.getClassLoader().getResourceAsStream(arch.getLibPath());
+ assert is != null;
+ File file = convertInputStreamToFile(is, arch);
+ System.load(file.getPath());
+ return true;
+ } catch (Throwable t) {
+ logger.info("Unable to load from jar: {}", String.valueOf(t));
+ }
+
+ return false;
+ }
+
+ private static File convertInputStreamToFile(InputStream is, Architecture arch)
+ throws IOException {
+ File tempFile = File.createTempFile("lib", arch.getFileExtension());
+ tempFile.deleteOnExit();
+
+ try (FileOutputStream os = new FileOutputStream(tempFile)) {
+ int read;
+ byte[] bytes = new byte[1024];
+
+ while ((read = is.read(bytes)) != -1) {
+ os.write(bytes, 0, read);
+ }
+ }
+
+ return tempFile;
}
/**
@@ -56,8 +179,6 @@ public final class LimboDB extends AbstractDB {
super(url, filePath);
}
- // WRAPPER FUNCTIONS ////////////////////////////////////////////
-
// TODO: add support for JNI
@Override
protected native long openUtf8(byte[] file, int openFlags) throws SQLException;
@@ -66,9 +187,6 @@ public final class LimboDB extends AbstractDB {
@Override
protected native void close0() throws SQLException;
- // TODO: add support for JNI
- native int execUtf8(byte[] sqlUtf8) throws SQLException;
-
// TODO: add support for JNI
@Override
public native void interrupt();
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboResultSet.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboResultSet.java
index c6cb8d00e..1884d4786 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboResultSet.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboResultSet.java
@@ -1,5 +1,6 @@
package org.github.tursodatabase.core;
+import java.sql.ResultSet;
import java.sql.SQLException;
import org.github.tursodatabase.annotations.Nullable;
import org.slf4j.Logger;
@@ -39,6 +40,20 @@ public class LimboResultSet {
this.statement = statement;
}
+ /**
+ * Consumes all the rows in this {@link ResultSet} until the {@link #next()} method returns
+ * `false`.
+ *
+ * @throws SQLException if the result set is not open or if an error occurs while iterating.
+ */
+ public void consumeAll() throws SQLException {
+ if (!open) {
+ throw new SQLException("The result set is not open");
+ }
+
+ while (next()) {}
+ }
+
/**
* Moves the cursor forward one row from its current position. A {@link LimboResultSet} cursor is
* initially positioned before the first fow; the first call to the method next makes
@@ -50,7 +65,11 @@ public class LimboResultSet {
* cursor can only move forward.
*/
public boolean next() throws SQLException {
- if (!open || isEmptyResultSet || pastLastRow) {
+ if (!open) {
+ throw new SQLException("The resultSet is not open");
+ }
+
+ if (isEmptyResultSet || pastLastRow) {
return false; // completed ResultSet
}
@@ -70,9 +89,6 @@ public class LimboResultSet {
}
pastLastRow = lastStepResult.isDone();
- if (pastLastRow) {
- open = false;
- }
return !pastLastRow;
}
@@ -97,6 +113,29 @@ public class LimboResultSet {
}
}
+ public void close() throws SQLException {
+ this.open = false;
+ }
+
+ // Note that columnIndex starts from 1
+ @Nullable
+ public Object get(int columnIndex) throws SQLException {
+ if (!this.isOpen()) {
+ throw new SQLException("ResultSet is not open");
+ }
+
+ if (this.lastStepResult == null || this.lastStepResult.getResult() == null) {
+ throw new SQLException("ResultSet is null");
+ }
+
+ final Object[] resultSet = this.lastStepResult.getResult();
+ if (columnIndex > resultSet.length || columnIndex < 0) {
+ throw new SQLException("columnIndex out of bound");
+ }
+
+ return resultSet[columnIndex - 1];
+ }
+
@Override
public String toString() {
return "LimboResultSet{"
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStatement.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStatement.java
index c749e27cc..fa660b67c 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStatement.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStatement.java
@@ -21,6 +21,8 @@ public class LimboStatement {
private final long statementPointer;
private final LimboResultSet resultSet;
+ private boolean closed;
+
// TODO: what if the statement we ran was DDL, update queries and etc. Should we still create a
// resultSet?
public LimboStatement(String sql, long statementPointer) {
@@ -53,6 +55,11 @@ public class LimboStatement {
return result;
}
+ /**
+ * Because Limbo supports async I/O, it is possible to return a {@link LimboStepResult} with
+ * {@link LimboStepResult#STEP_RESULT_ID_ROW}. However, this is handled by the native side, so you
+ * can expect that this method will not return a {@link LimboStepResult#STEP_RESULT_ID_ROW}.
+ */
@Nullable
private native LimboStepResult step(long stmtPointer) throws SQLException;
@@ -67,6 +74,30 @@ public class LimboStatement {
LimboExceptionUtils.throwLimboException(errorCode, errorMessageBytes);
}
+ /**
+ * Closes the current statement and releases any resources associated with it. This method calls
+ * the native `_close` method to perform the actual closing operation.
+ */
+ public void close() throws SQLException {
+ if (closed) {
+ return;
+ }
+ this.resultSet.close();
+ _close(statementPointer);
+ closed = true;
+ }
+
+ private native void _close(long statementPointer);
+
+ /**
+ * Checks if the statement is closed.
+ *
+ * @return true if the statement is closed, false otherwise.
+ */
+ public boolean isClosed() {
+ return closed;
+ }
+
@Override
public String toString() {
return "LimboStatement{"
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStepResult.java b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStepResult.java
index 93a1878aa..b82750b9a 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStepResult.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/core/LimboStepResult.java
@@ -47,6 +47,11 @@ public class LimboStepResult {
|| stepResultId == STEP_RESULT_ID_ERROR;
}
+ @Nullable
+ public Object[] getResult() {
+ return result;
+ }
+
@Override
public String toString() {
return "LimboStepResult{"
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java
index b17c6af36..3c32ffaf2 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Connection.java
@@ -83,13 +83,12 @@ public class JDBC4Connection extends LimboConnection {
@Override
public void close() throws SQLException {
- // TODO
+ super.close();
}
@Override
public boolean isClosed() throws SQLException {
- // TODO
- return false;
+ return super.isClosed();
}
@Override
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4ResultSet.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4ResultSet.java
index 867b2688e..ad3720b8c 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4ResultSet.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4ResultSet.java
@@ -3,10 +3,26 @@ package org.github.tursodatabase.jdbc4;
import java.io.InputStream;
import java.io.Reader;
import java.math.BigDecimal;
+import java.math.RoundingMode;
import java.net.URL;
-import java.sql.*;
+import java.sql.Array;
+import java.sql.Blob;
+import java.sql.Clob;
+import java.sql.Date;
+import java.sql.NClob;
+import java.sql.Ref;
+import java.sql.ResultSet;
+import java.sql.ResultSetMetaData;
+import java.sql.RowId;
+import java.sql.SQLException;
+import java.sql.SQLWarning;
+import java.sql.SQLXML;
+import java.sql.Statement;
+import java.sql.Time;
+import java.sql.Timestamp;
import java.util.Calendar;
import java.util.Map;
+import org.github.tursodatabase.annotations.Nullable;
import org.github.tursodatabase.annotations.SkipNullableCheck;
import org.github.tursodatabase.core.LimboResultSet;
@@ -25,7 +41,7 @@ public class JDBC4ResultSet implements ResultSet {
@Override
public void close() throws SQLException {
- // TODO
+ resultSet.close();
}
@Override
@@ -35,64 +51,99 @@ public class JDBC4ResultSet implements ResultSet {
}
@Override
+ @Nullable
public String getString(int columnIndex) throws SQLException {
- // TODO
- return "";
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return null;
+ }
+ return wrapTypeConversion(() -> (String) result);
}
@Override
public boolean getBoolean(int columnIndex) throws SQLException {
- // TODO
- return false;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return false;
+ }
+ return wrapTypeConversion(() -> (Long) result != 0);
}
@Override
public byte getByte(int columnIndex) throws SQLException {
- // TODO
- return 0;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return 0;
+ }
+ return wrapTypeConversion(() -> ((Long) result).byteValue());
}
@Override
public short getShort(int columnIndex) throws SQLException {
- // TODO
- return 0;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return 0;
+ }
+ return wrapTypeConversion(() -> ((Long) result).shortValue());
}
@Override
public int getInt(int columnIndex) throws SQLException {
- // TODO
- return 0;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return 0;
+ }
+ return wrapTypeConversion(() -> ((Long) result).intValue());
}
@Override
public long getLong(int columnIndex) throws SQLException {
- // TODO
- return 0;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return 0;
+ }
+ return wrapTypeConversion(() -> (long) result);
}
@Override
public float getFloat(int columnIndex) throws SQLException {
- // TODO
- return 0;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return 0;
+ }
+ return wrapTypeConversion(() -> ((Double) result).floatValue());
}
@Override
public double getDouble(int columnIndex) throws SQLException {
- // TODO
- return 0;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return 0;
+ }
+ return wrapTypeConversion(() -> (double) result);
}
+ // TODO: customize rounding mode?
@Override
- @SkipNullableCheck
+ @Nullable
public BigDecimal getBigDecimal(int columnIndex, int scale) throws SQLException {
- // TODO
- return null;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return null;
+ }
+ final double doubleResult = wrapTypeConversion(() -> (double) result);
+ final BigDecimal bigDecimalResult = BigDecimal.valueOf(doubleResult);
+ return bigDecimalResult.setScale(scale, RoundingMode.HALF_UP);
}
@Override
+ @Nullable
public byte[] getBytes(int columnIndex) throws SQLException {
- // TODO
- return new byte[0];
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return null;
+ }
+ return wrapTypeConversion(() -> (byte[]) result);
}
@Override
@@ -300,10 +351,14 @@ public class JDBC4ResultSet implements ResultSet {
}
@Override
- @SkipNullableCheck
+ @Nullable
public BigDecimal getBigDecimal(int columnIndex) throws SQLException {
- // TODO
- return null;
+ final Object result = resultSet.get(columnIndex);
+ if (result == null) {
+ return null;
+ }
+ final double doubleResult = wrapTypeConversion(() -> (double) result);
+ return BigDecimal.valueOf(doubleResult);
}
@Override
@@ -866,8 +921,7 @@ public class JDBC4ResultSet implements ResultSet {
@Override
public boolean isClosed() throws SQLException {
- // TODO
- return false;
+ return !resultSet.isOpen();
}
@Override
@@ -1127,7 +1181,16 @@ public class JDBC4ResultSet implements ResultSet {
return false;
}
- private SQLException throwNotSupportedException() {
- return new SQLFeatureNotSupportedException("Not implemented by the driver");
+ @FunctionalInterface
+ public interface ResultSetSupplier {
+ T get() throws Exception;
+ }
+
+ private T wrapTypeConversion(ResultSetSupplier supplier) throws SQLException {
+ try {
+ return supplier.get();
+ } catch (Exception e) {
+ throw new SQLException("Type conversion failed: " + e);
+ }
}
}
diff --git a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java
index eee4c95a3..3965e3cae 100644
--- a/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java
+++ b/bindings/java/src/main/java/org/github/tursodatabase/jdbc4/JDBC4Statement.java
@@ -19,6 +19,8 @@ public class JDBC4Statement implements Statement {
private final LimboConnection connection;
@Nullable private LimboStatement statement = null;
+ // Because JDBC4Statement has different life cycle in compared to LimboStatement, let's use this
+ // field to manage JDBC4Statement lifecycle
private boolean closed;
private boolean closeOnCompletion;
@@ -51,9 +53,21 @@ public class JDBC4Statement implements Statement {
this.resultSetHoldability = resultSetHoldability;
}
+ // TODO: should executeQuery run execute right after preparing the statement?
@Override
public ResultSet executeQuery(String sql) throws SQLException {
- execute(sql);
+ ensureOpen();
+ statement =
+ this.withConnectionTimeout(
+ () -> {
+ try {
+ // TODO: if sql is a readOnly query, do we still need the locks?
+ connectionLock.lock();
+ return connection.prepare(sql);
+ } finally {
+ connectionLock.unlock();
+ }
+ });
requireNonNull(statement, "statement should not be null after running execute method");
return new JDBC4ResultSet(statement.getResultSet());
@@ -65,9 +79,7 @@ public class JDBC4Statement implements Statement {
requireNonNull(statement, "statement should not be null after running execute method");
final LimboResultSet resultSet = statement.getResultSet();
- while (resultSet.isOpen()) {
- resultSet.next();
- }
+ resultSet.consumeAll();
// TODO: return update count;
return 0;
@@ -75,8 +87,14 @@ public class JDBC4Statement implements Statement {
@Override
public void close() throws SQLException {
- clearGeneratedKeys();
- internalClose();
+ if (closed) {
+ return;
+ }
+
+ if (this.statement != null) {
+ this.statement.close();
+ }
+
closed = true;
}
@@ -150,8 +168,7 @@ public class JDBC4Statement implements Statement {
*/
@Override
public boolean execute(String sql) throws SQLException {
- internalClose();
-
+ ensureOpen();
return this.withConnectionTimeout(
() -> {
try {
@@ -298,8 +315,7 @@ public class JDBC4Statement implements Statement {
@Override
public boolean isClosed() throws SQLException {
- // TODO
- return false;
+ return this.closed;
}
@Override
@@ -346,14 +362,6 @@ public class JDBC4Statement implements Statement {
return false;
}
- protected void internalClose() throws SQLException {
- // TODO
- }
-
- protected void clearGeneratedKeys() throws SQLException {
- // TODO
- }
-
protected void updateGeneratedKeys() throws SQLException {
// TODO
}
@@ -378,4 +386,10 @@ public class JDBC4Statement implements Statement {
protected interface SQLCallable {
T call() throws SQLException;
}
+
+ private void ensureOpen() throws SQLException {
+ if (closed) {
+ throw new SQLException("Statement is closed");
+ }
+ }
}
diff --git a/bindings/java/src/test/java/org/github/tursodatabase/core/LimboStatementTest.java b/bindings/java/src/test/java/org/github/tursodatabase/core/LimboStatementTest.java
new file mode 100644
index 000000000..fe274b07e
--- /dev/null
+++ b/bindings/java/src/test/java/org/github/tursodatabase/core/LimboStatementTest.java
@@ -0,0 +1,31 @@
+package org.github.tursodatabase.core;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+import java.util.Properties;
+import org.github.tursodatabase.TestUtils;
+import org.github.tursodatabase.jdbc4.JDBC4Connection;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class LimboStatementTest {
+
+ private JDBC4Connection connection;
+
+ @BeforeEach
+ void setUp() throws Exception {
+ String filePath = TestUtils.createTempFile();
+ String url = "jdbc:sqlite:" + filePath;
+ connection = new JDBC4Connection(url, filePath, new Properties());
+ }
+
+ @Test
+ void closing_statement_closes_related_resources() throws Exception {
+ LimboStatement stmt = connection.prepare("SELECT 1;");
+ stmt.execute();
+
+ stmt.close();
+ assertTrue(stmt.isClosed());
+ assertFalse(stmt.getResultSet().isOpen());
+ }
+}
diff --git a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java
index 60f6ee56e..1bc4fb526 100644
--- a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java
+++ b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ConnectionTest.java
@@ -65,4 +65,23 @@ class JDBC4ConnectionTest {
void prepare_simple_create_table() throws Exception {
connection.prepare("CREATE TABLE users (id INT PRIMARY KEY, username TEXT)");
}
+
+ @Test
+ void calling_close_multiple_times_throws_no_exception() throws Exception {
+ assertFalse(connection.isClosed());
+ connection.close();
+ assertTrue(connection.isClosed());
+ connection.close();
+ }
+
+ @Test
+ void calling_methods_on_closed_connection_should_throw_exception() throws Exception {
+ connection.close();
+ assertTrue(connection.isClosed());
+ assertThrows(
+ SQLException.class,
+ () ->
+ connection.createStatement(
+ ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY, -1));
+ }
}
diff --git a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ResultSetTest.java b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ResultSetTest.java
index f764a9361..ec79ce1eb 100644
--- a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ResultSetTest.java
+++ b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4ResultSetTest.java
@@ -1,14 +1,25 @@
package org.github.tursodatabase.jdbc4;
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import java.math.BigDecimal;
+import java.math.RoundingMode;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
+import java.util.stream.Stream;
import org.github.tursodatabase.TestUtils;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
class JDBC4ResultSetTest {
@@ -57,4 +68,449 @@ class JDBC4ResultSetTest {
// as well
assertFalse(resultSet.next());
}
+
+ @Test
+ void close_resultSet_test() throws Exception {
+ stmt.executeQuery("SELECT 1;");
+ ResultSet resultSet = stmt.getResultSet();
+
+ assertFalse(resultSet.isClosed());
+ resultSet.close();
+ assertTrue(resultSet.isClosed());
+ }
+
+ @Test
+ void calling_methods_on_closed_resultSet_should_throw_exception() throws Exception {
+ stmt.executeQuery("SELECT 1;");
+ ResultSet resultSet = stmt.getResultSet();
+ resultSet.close();
+ assertTrue(resultSet.isClosed());
+
+ assertThrows(SQLException.class, resultSet::next);
+ }
+
+ @Test
+ void test_getString() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_string (string_col TEXT);");
+ stmt.executeUpdate("INSERT INTO test_string (string_col) VALUES ('test');");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_string");
+ assertTrue(resultSet.next());
+ assertEquals("test", resultSet.getString(1));
+ }
+
+ @Test
+ void test_getString_returns_null_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (string_col TEXT);");
+ stmt.executeUpdate("INSERT INTO test_null (string_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertNull(resultSet.getString(1));
+ }
+
+ @Test
+ void test_getBoolean_true() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_boolean (boolean_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_boolean (boolean_col) VALUES (1);");
+ stmt.executeUpdate("INSERT INTO test_boolean (boolean_col) VALUES (2);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_boolean");
+
+ assertTrue(resultSet.next());
+ assertTrue(resultSet.getBoolean(1));
+
+ resultSet.next();
+ assertTrue(resultSet.getBoolean(1));
+ }
+
+ @Test
+ void test_getBoolean_false() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_boolean (boolean_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_boolean (boolean_col) VALUES (0);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_boolean");
+ assertTrue(resultSet.next());
+ assertFalse(resultSet.getBoolean(1));
+ }
+
+ @Test
+ void test_getBoolean_returns_false_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (boolean_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_null (boolean_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertFalse(resultSet.getBoolean(1));
+ }
+
+ @Test
+ void test_getByte() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_byte (byte_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_byte (byte_col) VALUES (1);");
+ stmt.executeUpdate("INSERT INTO test_byte (byte_col) VALUES (128);"); // Exceeds byte size
+ stmt.executeUpdate("INSERT INTO test_byte (byte_col) VALUES (-129);"); // Exceeds byte size
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_byte");
+
+ // Test value that fits within byte size
+ assertTrue(resultSet.next());
+ assertEquals(1, resultSet.getByte(1));
+
+ // Test value that exceeds byte size (positive overflow)
+ assertTrue(resultSet.next());
+ assertEquals(-128, resultSet.getByte(1)); // 128 overflows to -128
+
+ // Test value that exceeds byte size (negative overflow)
+ assertTrue(resultSet.next());
+ assertEquals(127, resultSet.getByte(1)); // -129 overflows to 127
+ }
+
+ @Test
+ void test_getByte_returns_zero_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (byte_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_null (byte_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertEquals(0, resultSet.getByte(1));
+ }
+
+ @Test
+ void test_getShort() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_short (short_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_short (short_col) VALUES (123);");
+ stmt.executeUpdate("INSERT INTO test_short (short_col) VALUES (32767);"); // Max short value
+ stmt.executeUpdate("INSERT INTO test_short (short_col) VALUES (-32768);"); // Min short value
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_short");
+
+ // Test typical short value
+ assertTrue(resultSet.next());
+ assertEquals(123, resultSet.getShort(1));
+
+ // Test maximum short value
+ assertTrue(resultSet.next());
+ assertEquals(32767, resultSet.getShort(1));
+
+ // Test minimum short value
+ assertTrue(resultSet.next());
+ assertEquals(-32768, resultSet.getShort(1));
+ }
+
+ @Test
+ void test_getShort_returns_zero_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (short_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_null (short_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertEquals(0, resultSet.getShort(1));
+ }
+
+ @Test
+ void test_getInt() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_int (int_col INT);");
+ stmt.executeUpdate("INSERT INTO test_int (int_col) VALUES (12345);");
+ stmt.executeUpdate("INSERT INTO test_int (int_col) VALUES (2147483647);"); // Max int value
+ stmt.executeUpdate("INSERT INTO test_int (int_col) VALUES (-2147483648);"); // Min int value
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_int");
+
+ // Test typical int value
+ assertTrue(resultSet.next());
+ assertEquals(12345, resultSet.getInt(1));
+
+ // Test maximum int value
+ assertTrue(resultSet.next());
+ assertEquals(2147483647, resultSet.getInt(1));
+
+ // Test minimum int value
+ assertTrue(resultSet.next());
+ assertEquals(-2147483648, resultSet.getInt(1));
+ }
+
+ @Test
+ void test_getInt_returns_zero_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (int_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_null (int_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertEquals(0, resultSet.getInt(1));
+ }
+
+ @Test
+ @Disabled("limbo has a bug which sees -9223372036854775808 as double")
+ void test_getLong() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_long (long_col BIGINT);");
+ stmt.executeUpdate("INSERT INTO test_long (long_col) VALUES (1234567890);");
+ stmt.executeUpdate(
+ "INSERT INTO test_long (long_col) VALUES (9223372036854775807);"); // Max long value
+ stmt.executeUpdate(
+ "INSERT INTO test_long (long_col) VALUES (-9223372036854775808);"); // Min long value
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_long");
+
+ // Test typical long value
+ assertEquals(1234567890L, resultSet.getLong(1));
+
+ // Test maximum long value
+ assertTrue(resultSet.next());
+ assertEquals(9223372036854775807L, resultSet.getLong(1));
+
+ // Test minimum long value
+ assertTrue(resultSet.next());
+ assertEquals(-9223372036854775808L, resultSet.getLong(1));
+ }
+
+ @Test
+ void test_getLong_returns_zero_no_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (long_col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_null (long_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertEquals(0L, resultSet.getLong(1));
+ }
+
+ @Test
+ void test_getFloat() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_float (float_col REAL);");
+ stmt.executeUpdate("INSERT INTO test_float (float_col) VALUES (1.23);");
+ stmt.executeUpdate(
+ "INSERT INTO test_float (float_col) VALUES (3.4028235E38);"); // Max float value
+ stmt.executeUpdate(
+ "INSERT INTO test_float (float_col) VALUES (1.4E-45);"); // Min positive float value
+ stmt.executeUpdate(
+ "INSERT INTO test_float (float_col) VALUES (-3.4028235E38);"); // Min negative float value
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_float");
+
+ // Test typical float value
+ assertTrue(resultSet.next());
+ assertEquals(1.23f, resultSet.getFloat(1), 0.0001);
+
+ // Test maximum float value
+ assertTrue(resultSet.next());
+ assertEquals(3.4028235E38f, resultSet.getFloat(1), 0.0001);
+
+ // Test minimum positive float value
+ assertTrue(resultSet.next());
+ assertEquals(1.4E-45f, resultSet.getFloat(1), 0.0001);
+
+ // Test minimum negative float value
+ assertTrue(resultSet.next());
+ assertEquals(-3.4028235E38f, resultSet.getFloat(1), 0.0001);
+ }
+
+ @Test
+ void test_getFloat_returns_zero_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (float_col REAL);");
+ stmt.executeUpdate("INSERT INTO test_null (float_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertEquals(0.0f, resultSet.getFloat(1), 0.0001);
+ }
+
+ @Test
+ void test_getDouble() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_double (double_col REAL);");
+ stmt.executeUpdate("INSERT INTO test_double (double_col) VALUES (1.234567);");
+ stmt.executeUpdate(
+ "INSERT INTO test_double (double_col) VALUES (1.7976931348623157E308);"); // Max double
+ // value
+ stmt.executeUpdate(
+ "INSERT INTO test_double (double_col) VALUES (4.9E-324);"); // Min positive double value
+ stmt.executeUpdate(
+ "INSERT INTO test_double (double_col) VALUES (-1.7976931348623157E308);"); // Min negative
+ // double value
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_double");
+
+ // Test typical double value
+ assertTrue(resultSet.next());
+ assertEquals(1.234567, resultSet.getDouble(1), 0.0001);
+
+ // Test maximum double value
+ assertTrue(resultSet.next());
+ assertEquals(1.7976931348623157E308, resultSet.getDouble(1), 0.0001);
+
+ // Test minimum positive double value
+ assertTrue(resultSet.next());
+ assertEquals(4.9E-324, resultSet.getDouble(1), 0.0001);
+
+ // Test minimum negative double value
+ assertTrue(resultSet.next());
+ assertEquals(-1.7976931348623157E308, resultSet.getDouble(1), 0.0001);
+ }
+
+ @Test
+ void test_getDouble_returns_zero_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (double_col REAL);");
+ stmt.executeUpdate("INSERT INTO test_null (double_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertEquals(0.0, resultSet.getDouble(1), 0.0001);
+ }
+
+ @Test
+ void test_getBigDecimal() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_bigdecimal (bigdecimal_col REAL);");
+ stmt.executeUpdate("INSERT INTO test_bigdecimal (bigdecimal_col) VALUES (12345.67);");
+ stmt.executeUpdate(
+ "INSERT INTO test_bigdecimal (bigdecimal_col) VALUES (1.7976931348623157E308);"); // Max
+ // double
+ // value
+ stmt.executeUpdate(
+ "INSERT INTO test_bigdecimal (bigdecimal_col) VALUES (4.9E-324);"); // Min positive double
+ // value
+ stmt.executeUpdate(
+ "INSERT INTO test_bigdecimal (bigdecimal_col) VALUES (-12345.67);"); // Negative value
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_bigdecimal");
+
+ // Test typical BigDecimal value
+ assertTrue(resultSet.next());
+ assertEquals(
+ new BigDecimal("12345.67").setScale(2, RoundingMode.HALF_UP),
+ resultSet.getBigDecimal(1, 2));
+
+ // Test maximum double value
+ assertTrue(resultSet.next());
+ assertEquals(
+ new BigDecimal("1.7976931348623157E308").setScale(10, RoundingMode.HALF_UP),
+ resultSet.getBigDecimal(1, 10));
+
+ // Test minimum positive double value
+ assertTrue(resultSet.next());
+ assertEquals(
+ new BigDecimal("4.9E-324").setScale(10, RoundingMode.HALF_UP),
+ resultSet.getBigDecimal(1, 10));
+
+ // Test negative BigDecimal value
+ assertTrue(resultSet.next());
+ assertEquals(
+ new BigDecimal("-12345.67").setScale(2, RoundingMode.HALF_UP),
+ resultSet.getBigDecimal(1, 2));
+ }
+
+ @Test
+ void test_getBigDecimal_returns_null_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (bigdecimal_col REAL);");
+ stmt.executeUpdate("INSERT INTO test_null (bigdecimal_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertNull(resultSet.getBigDecimal(1, 2));
+ }
+
+ @ParameterizedTest
+ @MethodSource("byteArrayProvider")
+ void test_getBytes(byte[] data) throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_bytes (bytes_col BLOB);");
+ executeDMLAndAssert(data);
+ }
+
+ private static Stream byteArrayProvider() {
+ return Stream.of(
+ "Hello".getBytes(), "world".getBytes(), new byte[0], new byte[] {0x00, (byte) 0xFF});
+ }
+
+ private void executeDMLAndAssert(byte[] data) throws SQLException {
+ // Convert byte array to hexadecimal string
+ StringBuilder hexString = new StringBuilder();
+ for (byte b : data) {
+ hexString.append(String.format("%02X", b));
+ }
+ // Execute DML statement
+ stmt.executeUpdate("INSERT INTO test_bytes (bytes_col) VALUES (X'" + hexString + "');");
+
+ // Assert the inserted data
+ ResultSet resultSet = stmt.executeQuery("SELECT bytes_col FROM test_bytes");
+ assertTrue(resultSet.next());
+ assertArrayEquals(data, resultSet.getBytes(1));
+ }
+
+ @Test
+ void test_getBytes_returns_null_on_null() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_null (bytes_col BLOB);");
+ stmt.executeUpdate("INSERT INTO test_null (bytes_col) VALUES (NULL);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_null");
+ assertTrue(resultSet.next());
+ assertNull(resultSet.getBytes(1));
+ }
+
+ @Test
+ void test_getXXX_methods_on_multiple_columns() throws Exception {
+ stmt.executeUpdate(
+ "CREATE TABLE test_integration ("
+ + "string_col TEXT, "
+ + "boolean_col INTEGER, "
+ + "byte_col INTEGER, "
+ + "short_col INTEGER, "
+ + "int_col INTEGER, "
+ + "long_col BIGINT, "
+ + "float_col REAL, "
+ + "double_col REAL, "
+ + "bigdecimal_col REAL, "
+ + "bytes_col BLOB);");
+
+ stmt.executeUpdate(
+ "INSERT INTO test_integration VALUES ("
+ + "'test', "
+ + "1, "
+ + "1, "
+ + "123, "
+ + "12345, "
+ + "1234567890, "
+ + "1.23, "
+ + "1.234567, "
+ + "12345.67, "
+ + "X'48656C6C6F');");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_integration");
+ assertTrue(resultSet.next());
+
+ // Verify each column
+ assertEquals("test", resultSet.getString(1));
+ assertTrue(resultSet.getBoolean(2));
+ assertEquals(1, resultSet.getByte(3));
+ assertEquals(123, resultSet.getShort(4));
+ assertEquals(12345, resultSet.getInt(5));
+ assertEquals(1234567890L, resultSet.getLong(6));
+ assertEquals(1.23f, resultSet.getFloat(7), 0.0001);
+ assertEquals(1.234567, resultSet.getDouble(8), 0.0001);
+ assertEquals(
+ new BigDecimal("12345.67").setScale(2, RoundingMode.HALF_UP),
+ resultSet.getBigDecimal(9, 2));
+ assertArrayEquals("Hello".getBytes(), resultSet.getBytes(10));
+ }
+
+ @Test
+ void test_invalidColumnIndex_outOfBounds() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_invalid (col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_invalid (col) VALUES (1);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_invalid");
+ assertTrue(resultSet.next());
+
+ // Test out-of-bounds column index
+ assertThrows(SQLException.class, () -> resultSet.getInt(2));
+ }
+
+ @Test
+ void test_invalidColumnIndex_negative() throws Exception {
+ stmt.executeUpdate("CREATE TABLE test_invalid (col INTEGER);");
+ stmt.executeUpdate("INSERT INTO test_invalid (col) VALUES (1);");
+
+ ResultSet resultSet = stmt.executeQuery("SELECT * FROM test_invalid");
+ assertTrue(resultSet.next());
+
+ // Test negative column index
+ assertThrows(SQLException.class, () -> resultSet.getInt(-1));
+ }
}
diff --git a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4StatementTest.java b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4StatementTest.java
index 2a837629d..a48cedea9 100644
--- a/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4StatementTest.java
+++ b/bindings/java/src/test/java/org/github/tursodatabase/jdbc4/JDBC4StatementTest.java
@@ -3,6 +3,7 @@ package org.github.tursodatabase.jdbc4;
import static org.junit.jupiter.api.Assertions.*;
import java.sql.ResultSet;
+import java.sql.SQLException;
import java.sql.Statement;
import java.util.Properties;
import org.github.tursodatabase.TestUtils;
@@ -51,4 +52,22 @@ class JDBC4StatementTest {
stmt.execute("INSERT INTO users VALUES (1, 'limbo');");
assertTrue(stmt.execute("SELECT * FROM users;"));
}
+
+ @Test
+ void close_statement_test() throws Exception {
+ stmt.close();
+ assertTrue(stmt.isClosed());
+ }
+
+ @Test
+ void double_close_is_no_op() throws SQLException {
+ stmt.close();
+ assertDoesNotThrow(() -> stmt.close());
+ }
+
+ @Test
+ void operations_on_closed_statement_should_throw_exception() throws Exception {
+ stmt.close();
+ assertThrows(SQLException.class, () -> stmt.execute("SELECT 1;"));
+ }
}
diff --git a/bindings/java/src/main/resources/logback.xml b/bindings/java/src/test/resources/logback.xml
similarity index 88%
rename from bindings/java/src/main/resources/logback.xml
rename to bindings/java/src/test/resources/logback.xml
index 5143dd837..1496a4b64 100644
--- a/bindings/java/src/main/resources/logback.xml
+++ b/bindings/java/src/test/resources/logback.xml
@@ -5,7 +5,7 @@
-
+
diff --git a/bindings/python/src/lib.rs b/bindings/python/src/lib.rs
index 085d71b3b..16afc5494 100644
--- a/bindings/python/src/lib.rs
+++ b/bindings/python/src/lib.rs
@@ -101,16 +101,17 @@ impl Cursor {
// For DDL and DML statements,
// we need to execute the statement immediately
if stmt_is_ddl || stmt_is_dml {
- while stmt
- .borrow_mut()
- .step()
- .map_err(|e| PyErr::new::(format!("Step error: {:?}", e)))?
- .eq(&limbo_core::StepResult::IO)
- {
- self.conn
- .io
- .run_once()
- .map_err(|e| PyErr::new::(format!("IO error: {:?}", e)))?;
+ loop {
+ match stmt.borrow_mut().step().map_err(|e| {
+ PyErr::new::(format!("Step error: {:?}", e))
+ })? {
+ limbo_core::StepResult::IO => {
+ self.conn.io.run_once().map_err(|e| {
+ PyErr::new::(format!("IO error: {:?}", e))
+ })?;
+ }
+ _ => break,
+ }
}
}
diff --git a/bindings/wasm/integration-tests/package-lock.json b/bindings/wasm/integration-tests/package-lock.json
index f2886d47d..ea731abaf 100644
--- a/bindings/wasm/integration-tests/package-lock.json
+++ b/bindings/wasm/integration-tests/package-lock.json
@@ -15,7 +15,7 @@
},
"..": {
"name": "limbo-wasm",
- "version": "0.0.13",
+ "version": "0.0.14",
"license": "MIT",
"devDependencies": {
"@playwright/test": "^1.49.1",
diff --git a/bindings/wasm/lib.rs b/bindings/wasm/lib.rs
index 1558f09d8..9b34fcb57 100644
--- a/bindings/wasm/lib.rs
+++ b/bindings/wasm/lib.rs
@@ -192,7 +192,7 @@ fn to_js_value(value: limbo_core::Value) -> JsValue {
}
limbo_core::Value::Float(f) => JsValue::from(f),
limbo_core::Value::Text(t) => JsValue::from_str(t),
- limbo_core::Value::Blob(b) => js_sys::Uint8Array::from(b.as_slice()).into(),
+ limbo_core::Value::Blob(b) => js_sys::Uint8Array::from(b).into(),
}
}
diff --git a/bindings/wasm/package-lock.json b/bindings/wasm/package-lock.json
index 83309fcb9..930d9255e 100644
--- a/bindings/wasm/package-lock.json
+++ b/bindings/wasm/package-lock.json
@@ -1,12 +1,12 @@
{
"name": "limbo-wasm",
- "version": "0.0.13",
+ "version": "0.0.14",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "limbo-wasm",
- "version": "0.0.13",
+ "version": "0.0.14",
"license": "MIT",
"devDependencies": {
"@playwright/test": "^1.49.1",
diff --git a/bindings/wasm/package.json b/bindings/wasm/package.json
index 7dfb91516..2265a7799 100644
--- a/bindings/wasm/package.json
+++ b/bindings/wasm/package.json
@@ -3,7 +3,7 @@
"collaborators": [
"the Limbo authors"
],
- "version": "0.0.13",
+ "version": "0.0.14",
"license": "MIT",
"repository": {
"type": "git",
diff --git a/bindings/wasm/test-limbo-pkg/package.json b/bindings/wasm/test-limbo-pkg/package.json
index 424bcede1..7c10b8c28 100644
--- a/bindings/wasm/test-limbo-pkg/package.json
+++ b/bindings/wasm/test-limbo-pkg/package.json
@@ -3,7 +3,7 @@
"private": true,
"type": "module",
"dependencies": {
- "limbo-wasm": "limbo-wasm@0.0.13"
+ "limbo-wasm": "limbo-wasm@0.0.14"
},
"scripts": {
"dev": "vite"
diff --git a/core/Cargo.toml b/core/Cargo.toml
index c378d8bcc..f2bba5f59 100644
--- a/core/Cargo.toml
+++ b/core/Cargo.toml
@@ -14,15 +14,19 @@ name = "limbo_core"
path = "lib.rs"
[features]
-default = ["fs", "json", "uuid", "io_uring"]
+default = ["fs", "json", "uuid", "vector", "io_uring", "time"]
fs = []
json = [
"dep:jsonb",
"dep:pest",
"dep:pest_derive",
]
-uuid = ["dep:uuid"]
+uuid = ["limbo_uuid/static"]
+vector = ["limbo_vector/static"]
io_uring = ["dep:io-uring", "rustix/io_uring"]
+percentile = ["limbo_percentile/static"]
+regexp = ["limbo_regexp/static"]
+time = ["limbo_time/static"]
[target.'cfg(target_os = "linux")'.dependencies]
io-uring = { version = "0.6.1", optional = true }
@@ -33,6 +37,7 @@ rustix = "0.38.34"
[target.'cfg(not(target_family = "wasm"))'.dependencies]
mimalloc = { version = "*", default-features = false }
+libloading = "0.8.6"
[dependencies]
limbo_ext = { path = "../extensions/core" }
@@ -57,9 +62,20 @@ pest_derive = { version = "2.0", optional = true }
rand = "0.8.5"
bumpalo = { version = "3.16.0", features = ["collections", "boxed"] }
limbo_macros = { path = "../macros" }
-uuid = { version = "1.11.0", features = ["v4", "v7"], optional = true }
+limbo_uuid = { path = "../extensions/uuid", optional = true, features = ["static"] }
+limbo_vector = { path = "../extensions/vector", optional = true, features = ["static"] }
+limbo_regexp = { path = "../extensions/regexp", optional = true, features = ["static"] }
+limbo_percentile = { path = "../extensions/percentile", optional = true, features = ["static"] }
+limbo_time = { path = "../extensions/time", optional = true, features = ["static"] }
miette = "7.4.0"
-libloading = "0.8.6"
+strum = "0.26"
+parking_lot = "0.12.3"
+tracing = "0.1.41"
+crossbeam-skiplist = "0.1.3"
+
+[build-dependencies]
+chrono = "0.4.38"
+built = { version = "0.7.5", features = ["git2", "chrono"] }
[target.'cfg(not(target_family = "windows"))'.dev-dependencies]
pprof = { version = "0.14.0", features = ["criterion", "flamegraph"] }
@@ -78,3 +94,7 @@ tempfile = "3.8.0"
[[bench]]
name = "benchmark"
harness = false
+
+[[bench]]
+name = "mvcc_benchmark"
+harness = false
diff --git a/core/benches/benchmark.rs b/core/benches/benchmark.rs
index 9858a0c56..f25d48144 100644
--- a/core/benches/benchmark.rs
+++ b/core/benches/benchmark.rs
@@ -1,186 +1,145 @@
-use criterion::{criterion_group, criterion_main, Criterion, Throughput};
+use criterion::{black_box, criterion_group, criterion_main, BenchmarkId, Criterion};
use limbo_core::{Database, PlatformIO, IO};
use pprof::criterion::{Output, PProfProfiler};
use std::sync::Arc;
-fn bench(c: &mut Criterion) {
- limbo_bench(c);
-
- // https://github.com/penberg/limbo/issues/174
- // The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
- if std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_ok() {
- return;
- }
-
- rusqlite_bench(c)
+fn rusqlite_open() -> rusqlite::Connection {
+ let sqlite_conn = rusqlite::Connection::open("../testing/testing.db").unwrap();
+ sqlite_conn
+ .pragma_update(None, "locking_mode", "EXCLUSIVE")
+ .unwrap();
+ sqlite_conn
}
-fn limbo_bench(criterion: &mut Criterion) {
- let mut group = criterion.benchmark_group("limbo");
- group.throughput(Throughput::Elements(1));
+fn bench(criterion: &mut Criterion) {
+ // https://github.com/penberg/limbo/issues/174
+ // The rusqlite benchmark crashes on Mac M1 when using the flamegraph features
+ let enable_rusqlite = std::env::var("DISABLE_RUSQLITE_BENCHMARK").is_err();
+
#[allow(clippy::arc_with_non_send_sync)]
let io = Arc::new(PlatformIO::new().unwrap());
let db = Database::open_file(io.clone(), "../testing/testing.db").unwrap();
- let conn = db.connect();
+ let limbo_conn = db.connect();
- group.bench_function("Prepare statement: 'SELECT 1'", |b| {
- b.iter(|| {
- conn.prepare("SELECT 1").unwrap();
+ let queries = [
+ "SELECT 1",
+ "SELECT * FROM users LIMIT 1",
+ "SELECT first_name, count(1) FROM users GROUP BY first_name HAVING count(1) > 1 ORDER BY count(1) LIMIT 1",
+ ];
+
+ for query in queries.iter() {
+ let mut group = criterion.benchmark_group(format!("Prepare `{}`", query));
+
+ group.bench_with_input(BenchmarkId::new("Limbo", query), query, |b, query| {
+ b.iter(|| {
+ limbo_conn.prepare(query).unwrap();
+ });
});
- });
- group.bench_function("Prepare statement: 'SELECT * FROM users LIMIT 1'", |b| {
- b.iter(|| {
- conn.prepare("SELECT * FROM users LIMIT 1").unwrap();
+ if enable_rusqlite {
+ let sqlite_conn = rusqlite_open();
+
+ group.bench_with_input(BenchmarkId::new("Sqlite3", query), query, |b, query| {
+ b.iter(|| {
+ sqlite_conn.prepare(query).unwrap();
+ });
+ });
+ }
+
+ group.finish();
+ }
+
+ let mut group = criterion.benchmark_group("Execute `SELECT * FROM users LIMIT ?`");
+
+ for i in [1, 10, 50, 100] {
+ group.bench_with_input(BenchmarkId::new("Limbo", i), &i, |b, i| {
+ // TODO: LIMIT doesn't support query parameters.
+ let mut stmt = limbo_conn
+ .prepare(format!("SELECT * FROM users LIMIT {}", *i))
+ .unwrap();
+ let io = io.clone();
+ b.iter(|| {
+ loop {
+ match stmt.step().unwrap() {
+ limbo_core::StepResult::Row(row) => {
+ black_box(row);
+ }
+ limbo_core::StepResult::IO => {
+ let _ = io.run_once();
+ }
+ limbo_core::StepResult::Done => {
+ break;
+ }
+ limbo_core::StepResult::Interrupt | limbo_core::StepResult::Busy => {
+ unreachable!();
+ }
+ }
+ }
+ stmt.reset();
+ });
});
- });
- group.bench_function("Prepare statement: 'SELECT first_name, count(1) FROM users GROUP BY first_name HAVING count(1) > 1 ORDER BY count(1) LIMIT 1'", |b| {
- b.iter(|| {
- conn.prepare("SELECT first_name, count(1) FROM users GROUP BY first_name HAVING count(1) > 1 ORDER BY count(1) LIMIT 1").unwrap();
- });
- });
+ if enable_rusqlite {
+ let sqlite_conn = rusqlite_open();
- let mut stmt = conn.prepare("SELECT 1").unwrap();
- group.bench_function("Execute prepared statement: 'SELECT 1'", |b| {
+ group.bench_with_input(BenchmarkId::new("Sqlite3", i), &i, |b, i| {
+ // TODO: Use parameters once we fix the above.
+ let mut stmt = sqlite_conn
+ .prepare(&format!("SELECT * FROM users LIMIT {}", *i))
+ .unwrap();
+ b.iter(|| {
+ let mut rows = stmt.raw_query();
+ while let Some(row) = rows.next().unwrap() {
+ black_box(row);
+ }
+ });
+ });
+ }
+ }
+
+ group.finish();
+
+ let mut group = criterion.benchmark_group("Execute `SELECT 1`");
+
+ group.bench_function("Limbo", |b| {
+ let mut stmt = limbo_conn.prepare("SELECT 1").unwrap();
let io = io.clone();
b.iter(|| {
- let mut rows = stmt.query().unwrap();
- match rows.step().unwrap() {
- limbo_core::StepResult::Row(row) => {
- assert_eq!(row.get::(0).unwrap(), 1);
- }
- limbo_core::StepResult::IO => {
- io.run_once().unwrap();
- }
- limbo_core::StepResult::Interrupt => {
- unreachable!();
- }
- limbo_core::StepResult::Done => {
- unreachable!();
- }
- limbo_core::StepResult::Busy => {
- unreachable!();
+ loop {
+ match stmt.step().unwrap() {
+ limbo_core::StepResult::Row(row) => {
+ black_box(row);
+ }
+ limbo_core::StepResult::IO => {
+ let _ = io.run_once();
+ }
+ limbo_core::StepResult::Done => {
+ break;
+ }
+ limbo_core::StepResult::Interrupt | limbo_core::StepResult::Busy => {
+ unreachable!();
+ }
}
}
stmt.reset();
});
});
- let mut stmt = conn.prepare("SELECT * FROM users LIMIT 1").unwrap();
- group.bench_function(
- "Execute prepared statement: 'SELECT * FROM users LIMIT 1'",
- |b| {
- let io = io.clone();
+ if enable_rusqlite {
+ let sqlite_conn = rusqlite_open();
+
+ group.bench_function("Sqlite3", |b| {
+ let mut stmt = sqlite_conn.prepare("SELECT 1").unwrap();
b.iter(|| {
- let mut rows = stmt.query().unwrap();
- match rows.step().unwrap() {
- limbo_core::StepResult::Row(row) => {
- assert_eq!(row.get::(0).unwrap(), 1);
- }
- limbo_core::StepResult::IO => {
- io.run_once().unwrap();
- }
- limbo_core::StepResult::Interrupt => {
- unreachable!();
- }
- limbo_core::StepResult::Done => {
- unreachable!();
- }
- limbo_core::StepResult::Busy => {
- unreachable!()
- }
+ let mut rows = stmt.raw_query();
+ while let Some(row) = rows.next().unwrap() {
+ black_box(row);
}
- stmt.reset();
});
- },
- );
-
- let mut stmt = conn.prepare("SELECT * FROM users LIMIT 100").unwrap();
- group.bench_function(
- "Execute prepared statement: 'SELECT * FROM users LIMIT 100'",
- |b| {
- let io = io.clone();
- b.iter(|| {
- let mut rows = stmt.query().unwrap();
- match rows.step().unwrap() {
- limbo_core::StepResult::Row(row) => {
- assert_eq!(row.get::(0).unwrap(), 1);
- }
- limbo_core::StepResult::IO => {
- io.run_once().unwrap();
- }
- limbo_core::StepResult::Interrupt => {
- unreachable!();
- }
- limbo_core::StepResult::Done => {
- unreachable!();
- }
- limbo_core::StepResult::Busy => {
- unreachable!()
- }
- }
- stmt.reset();
- });
- },
- );
-}
-
-fn rusqlite_bench(criterion: &mut Criterion) {
- let mut group = criterion.benchmark_group("rusqlite");
- group.throughput(Throughput::Elements(1));
-
- let conn = rusqlite::Connection::open("../testing/testing.db").unwrap();
-
- conn.pragma_update(None, "locking_mode", "EXCLUSIVE")
- .unwrap();
- group.bench_function("Prepare statement: 'SELECT 1'", |b| {
- b.iter(|| {
- conn.prepare("SELECT 1").unwrap();
});
- });
+ }
- group.bench_function("Prepare statement: 'SELECT * FROM users LIMIT 1'", |b| {
- b.iter(|| {
- conn.prepare("SELECT * FROM users LIMIT 1").unwrap();
- });
- });
-
- let mut stmt = conn.prepare("SELECT 1").unwrap();
- group.bench_function("Execute prepared statement: 'SELECT 1'", |b| {
- b.iter(|| {
- let mut rows = stmt.query(()).unwrap();
- let row = rows.next().unwrap().unwrap();
- let val: i64 = row.get(0).unwrap();
- assert_eq!(val, 1);
- });
- });
-
- let mut stmt = conn.prepare("SELECT * FROM users LIMIT 1").unwrap();
- group.bench_function(
- "Execute prepared statement: 'SELECT * FROM users LIMIT 1'",
- |b| {
- b.iter(|| {
- let mut rows = stmt.query(()).unwrap();
- let row = rows.next().unwrap().unwrap();
- let id: i64 = row.get(0).unwrap();
- assert_eq!(id, 1);
- });
- },
- );
-
- let mut stmt = conn.prepare("SELECT * FROM users LIMIT 100").unwrap();
- group.bench_function(
- "Execute prepared statement: 'SELECT * FROM users LIMIT 100'",
- |b| {
- b.iter(|| {
- let mut rows = stmt.query(()).unwrap();
- let row = rows.next().unwrap().unwrap();
- let id: i64 = row.get(0).unwrap();
- assert_eq!(id, 1);
- });
- },
- );
+ group.finish();
}
criterion_group! {
diff --git a/core/benches/mvcc_benchmark.rs b/core/benches/mvcc_benchmark.rs
new file mode 100644
index 000000000..899c8b82d
--- /dev/null
+++ b/core/benches/mvcc_benchmark.rs
@@ -0,0 +1,129 @@
+use criterion::async_executor::FuturesExecutor;
+use criterion::{criterion_group, criterion_main, Criterion, Throughput};
+use limbo_core::mvcc::clock::LocalClock;
+use limbo_core::mvcc::database::{Database, Row, RowID};
+use pprof::criterion::{Output, PProfProfiler};
+
+fn bench_db() -> Database {
+ let clock = LocalClock::default();
+ let storage = limbo_core::mvcc::persistent_storage::Storage::new_noop();
+ Database::new(clock, storage)
+}
+
+fn bench(c: &mut Criterion) {
+ let mut group = c.benchmark_group("mvcc-ops-throughput");
+ group.throughput(Throughput::Elements(1));
+
+ let db = bench_db();
+ group.bench_function("begin_tx + rollback_tx", |b| {
+ b.to_async(FuturesExecutor).iter(|| async {
+ let tx_id = db.begin_tx();
+ db.rollback_tx(tx_id)
+ })
+ });
+
+ let db = bench_db();
+ group.bench_function("begin_tx + commit_tx", |b| {
+ b.to_async(FuturesExecutor).iter(|| async {
+ let tx_id = db.begin_tx();
+ db.commit_tx(tx_id)
+ })
+ });
+
+ let db = bench_db();
+ group.bench_function("begin_tx-read-commit_tx", |b| {
+ b.to_async(FuturesExecutor).iter(|| async {
+ let tx_id = db.begin_tx();
+ db.read(
+ tx_id,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ db.commit_tx(tx_id)
+ })
+ });
+
+ let db = bench_db();
+ group.bench_function("begin_tx-update-commit_tx", |b| {
+ b.to_async(FuturesExecutor).iter(|| async {
+ let tx_id = db.begin_tx();
+ db.update(
+ tx_id,
+ Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ },
+ )
+ .unwrap();
+ db.commit_tx(tx_id)
+ })
+ });
+
+ let db = bench_db();
+ let tx = db.begin_tx();
+ db.insert(
+ tx,
+ Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ },
+ )
+ .unwrap();
+ group.bench_function("read", |b| {
+ b.to_async(FuturesExecutor).iter(|| async {
+ db.read(
+ tx,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ })
+ });
+
+ let db = bench_db();
+ let tx = db.begin_tx();
+ db.insert(
+ tx,
+ Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ },
+ )
+ .unwrap();
+ group.bench_function("update", |b| {
+ b.to_async(FuturesExecutor).iter(|| async {
+ db.update(
+ tx,
+ Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ },
+ )
+ .unwrap();
+ })
+ });
+}
+
+criterion_group! {
+ name = benches;
+ config = Criterion::default().with_profiler(PProfProfiler::new(100, Output::Flamegraph(None)));
+ targets = bench
+}
+criterion_main!(benches);
diff --git a/core/build.rs b/core/build.rs
new file mode 100644
index 000000000..50afee6bf
--- /dev/null
+++ b/core/build.rs
@@ -0,0 +1,21 @@
+use std::fs;
+use std::path::PathBuf;
+
+fn main() {
+ let out_dir = PathBuf::from(std::env::var("OUT_DIR").unwrap());
+ let built_file = out_dir.join("built.rs");
+
+ built::write_built_file().expect("Failed to acquire build-time information");
+
+ // So that we don't have to transform at runtime
+ let sqlite_date = chrono::Utc::now().format("%Y-%m-%d %H:%M:%S").to_string();
+ fs::write(
+ &built_file,
+ format!(
+ "{}\npub const BUILT_TIME_SQLITE: &str = \"{}\";\n",
+ fs::read_to_string(&built_file).unwrap(),
+ sqlite_date
+ ),
+ )
+ .expect("Failed to append to built file");
+}
diff --git a/core/error.rs b/core/error.rs
index ca495eb99..cfe1a827d 100644
--- a/core/error.rs
+++ b/core/error.rs
@@ -39,12 +39,18 @@ pub enum LimboError {
InvalidTime(String),
#[error("Modifier parsing error: {0}")]
InvalidModifier(String),
+ #[error("Invalid argument supplied: {0}")]
+ InvalidArgument(String),
+ #[error("Invalid formatter supplied: {0}")]
+ InvalidFormatter(String),
#[error("Runtime error: {0}")]
Constraint(String),
#[error("Extension error: {0}")]
ExtensionError(String),
#[error("Unbound parameter at index {0}")]
Unbound(NonZero),
+ #[error("Runtime error: integer overflow")]
+ IntegerOverflow,
}
#[macro_export]
diff --git a/core/ext/mod.rs b/core/ext/mod.rs
index cf3fa6109..8a9212556 100644
--- a/core/ext/mod.rs
+++ b/core/ext/mod.rs
@@ -73,4 +73,29 @@ impl Database {
register_aggregate_function,
}
}
+
+ pub fn register_builtins(&self) -> Result<(), String> {
+ let ext_api = self.build_limbo_ext();
+ #[cfg(feature = "uuid")]
+ if unsafe { !limbo_uuid::register_extension_static(&ext_api).is_ok() } {
+ return Err("Failed to register uuid extension".to_string());
+ }
+ #[cfg(feature = "vector")]
+ if unsafe { !limbo_vector::register_extension_static(&ext_api).is_ok() } {
+ return Err("Failed to register vector extension".to_string());
+ }
+ #[cfg(feature = "percentile")]
+ if unsafe { !limbo_percentile::register_extension_static(&ext_api).is_ok() } {
+ return Err("Failed to register percentile extension".to_string());
+ }
+ #[cfg(feature = "regexp")]
+ if unsafe { !limbo_regexp::register_extension_static(&ext_api).is_ok() } {
+ return Err("Failed to register regexp extension".to_string());
+ }
+ #[cfg(feature = "time")]
+ if unsafe { !limbo_time::register_extension_static(&ext_api).is_ok() } {
+ return Err("Failed to register time extension".to_string());
+ }
+ Ok(())
+ }
}
diff --git a/core/function.rs b/core/function.rs
index 1e5386696..8fbe9cb7e 100644
--- a/core/function.rs
+++ b/core/function.rs
@@ -80,6 +80,10 @@ pub enum JsonFunc {
JsonType,
JsonErrorPosition,
JsonValid,
+ JsonPatch,
+ JsonRemove,
+ JsonPretty,
+ JsonSet,
}
#[cfg(feature = "json")]
@@ -99,6 +103,10 @@ impl Display for JsonFunc {
Self::JsonType => "json_type".to_string(),
Self::JsonErrorPosition => "json_error_position".to_string(),
Self::JsonValid => "json_valid".to_string(),
+ Self::JsonPatch => "json_patch".to_string(),
+ Self::JsonRemove => "json_remove".to_string(),
+ Self::JsonPretty => "json_pretty".to_string(),
+ Self::JsonSet => "json_set".to_string(),
}
)
}
@@ -206,6 +214,7 @@ pub enum ScalarFunc {
Unicode,
Quote,
SqliteVersion,
+ SqliteSourceId,
UnixEpoch,
JulianDay,
Hex,
@@ -216,6 +225,7 @@ pub enum ScalarFunc {
#[cfg(not(target_family = "wasm"))]
LoadExtension,
StrfTime,
+ Printf,
}
impl Display for ScalarFunc {
@@ -257,6 +267,7 @@ impl Display for ScalarFunc {
Self::Unicode => "unicode".to_string(),
Self::Quote => "quote".to_string(),
Self::SqliteVersion => "sqlite_version".to_string(),
+ Self::SqliteSourceId => "sqlite_source_id".to_string(),
Self::JulianDay => "julianday".to_string(),
Self::UnixEpoch => "unixepoch".to_string(),
Self::Hex => "hex".to_string(),
@@ -268,6 +279,7 @@ impl Display for ScalarFunc {
#[cfg(not(target_family = "wasm"))]
Self::LoadExtension => "load_extension".to_string(),
Self::StrfTime => "strftime".to_string(),
+ Self::Printf => "printf".to_string(),
};
write!(f, "{}", str)
}
@@ -506,6 +518,7 @@ impl Func {
"unicode" => Ok(Self::Scalar(ScalarFunc::Unicode)),
"quote" => Ok(Self::Scalar(ScalarFunc::Quote)),
"sqlite_version" => Ok(Self::Scalar(ScalarFunc::SqliteVersion)),
+ "sqlite_source_id" => Ok(Self::Scalar(ScalarFunc::SqliteSourceId)),
"replace" => Ok(Self::Scalar(ScalarFunc::Replace)),
#[cfg(feature = "json")]
"json" => Ok(Self::Json(JsonFunc::Json)),
@@ -523,6 +536,14 @@ impl Func {
"json_error_position" => Ok(Self::Json(JsonFunc::JsonErrorPosition)),
#[cfg(feature = "json")]
"json_valid" => Ok(Self::Json(JsonFunc::JsonValid)),
+ #[cfg(feature = "json")]
+ "json_patch" => Ok(Self::Json(JsonFunc::JsonPatch)),
+ #[cfg(feature = "json")]
+ "json_remove" => Ok(Self::Json(JsonFunc::JsonRemove)),
+ #[cfg(feature = "json")]
+ "json_pretty" => Ok(Self::Json(JsonFunc::JsonPretty)),
+ #[cfg(feature = "json")]
+ "json_set" => Ok(Self::Json(JsonFunc::JsonSet)),
"unixepoch" => Ok(Self::Scalar(ScalarFunc::UnixEpoch)),
"julianday" => Ok(Self::Scalar(ScalarFunc::JulianDay)),
"hex" => Ok(Self::Scalar(ScalarFunc::Hex)),
@@ -561,6 +582,7 @@ impl Func {
#[cfg(not(target_family = "wasm"))]
"load_extension" => Ok(Self::Scalar(ScalarFunc::LoadExtension)),
"strftime" => Ok(Self::Scalar(ScalarFunc::StrfTime)),
+ "printf" => Ok(Self::Scalar(ScalarFunc::Printf)),
_ => crate::bail_parse_error!("no such function: {}", name),
}
}
diff --git a/core/info.rs b/core/info.rs
new file mode 100644
index 000000000..908ddca7b
--- /dev/null
+++ b/core/info.rs
@@ -0,0 +1,3 @@
+pub mod build {
+ include!(concat!(env!("OUT_DIR"), "/built.rs"));
+}
diff --git a/core/io/memory.rs b/core/io/memory.rs
index 6fc960e13..18decf78a 100644
--- a/core/io/memory.rs
+++ b/core/io/memory.rs
@@ -3,15 +3,15 @@ use crate::Result;
use log::debug;
use std::{
- cell::{RefCell, RefMut},
+ cell::{Cell, RefCell, UnsafeCell},
collections::BTreeMap,
rc::Rc,
sync::Arc,
};
pub struct MemoryIO {
- pages: RefCell>,
- size: RefCell,
+ pages: UnsafeCell>,
+ size: Cell,
}
// TODO: page size flag
@@ -23,23 +23,23 @@ impl MemoryIO {
pub fn new() -> Result> {
debug!("Using IO backend 'memory'");
Ok(Arc::new(Self {
- pages: RefCell::new(BTreeMap::new()),
- size: RefCell::new(0),
+ pages: BTreeMap::new().into(),
+ size: 0.into(),
}))
}
- fn get_or_allocate_page(&self, page_no: usize) -> RefMut {
- let pages = self.pages.borrow_mut();
- RefMut::map(pages, |p| {
- p.entry(page_no).or_insert_with(|| Box::new([0; PAGE_SIZE]))
- })
+ #[allow(clippy::mut_from_ref)]
+ fn get_or_allocate_page(&self, page_no: usize) -> &mut MemPage {
+ unsafe {
+ let pages = &mut *self.pages.get();
+ pages
+ .entry(page_no)
+ .or_insert_with(|| Box::new([0; PAGE_SIZE]))
+ }
}
- fn get_page(&self, page_no: usize) -> Option> {
- match RefMut::filter_map(self.pages.borrow_mut(), |pages| pages.get_mut(&page_no)) {
- Ok(page) => Some(page),
- Err(_) => None,
- }
+ fn get_page(&self, page_no: usize) -> Option<&MemPage> {
+ unsafe { (*self.pages.get()).get(&page_no) }
}
}
@@ -71,7 +71,6 @@ pub struct MemoryFile {
}
impl File for MemoryFile {
- // no-ops
fn lock_file(&self, _exclusive: bool) -> Result<()> {
Ok(())
}
@@ -90,7 +89,7 @@ impl File for MemoryFile {
return Ok(());
}
- let file_size = *self.io.size.borrow();
+ let file_size = self.io.size.get();
if pos >= file_size {
c.complete(0);
return Ok(());
@@ -108,15 +107,10 @@ impl File for MemoryFile {
let page_offset = offset % PAGE_SIZE;
let bytes_to_read = remaining.min(PAGE_SIZE - page_offset);
if let Some(page) = self.io.get_page(page_no) {
- {
- let page_data = &*page;
- read_buf.as_mut_slice()[buf_offset..buf_offset + bytes_to_read]
- .copy_from_slice(&page_data[page_offset..page_offset + bytes_to_read]);
- }
+ read_buf.as_mut_slice()[buf_offset..buf_offset + bytes_to_read]
+ .copy_from_slice(&page[page_offset..page_offset + bytes_to_read]);
} else {
- for b in &mut read_buf.as_mut_slice()[buf_offset..buf_offset + bytes_to_read] {
- *b = 0;
- }
+ read_buf.as_mut_slice()[buf_offset..buf_offset + bytes_to_read].fill(0);
}
offset += bytes_to_read;
@@ -147,7 +141,7 @@ impl File for MemoryFile {
let bytes_to_write = remaining.min(PAGE_SIZE - page_offset);
{
- let mut page = self.io.get_or_allocate_page(page_no);
+ let page = self.io.get_or_allocate_page(page_no);
page[page_offset..page_offset + bytes_to_write]
.copy_from_slice(&data[buf_offset..buf_offset + bytes_to_write]);
}
@@ -157,10 +151,9 @@ impl File for MemoryFile {
remaining -= bytes_to_write;
}
- {
- let mut size = self.io.size.borrow_mut();
- *size = (*size).max(pos + buf_len);
- }
+ self.io
+ .size
+ .set(core::cmp::max(pos + buf_len, self.io.size.get()));
c.complete(buf_len as i32);
Ok(())
@@ -173,7 +166,7 @@ impl File for MemoryFile {
}
fn size(&self) -> Result {
- Ok(*self.io.size.borrow() as u64)
+ Ok(self.io.size.get() as u64)
}
}
diff --git a/core/json/de.rs b/core/json/de.rs
index 14927bb18..6cc3fb333 100644
--- a/core/json/de.rs
+++ b/core/json/de.rs
@@ -504,3 +504,59 @@ impl<'de> de::VariantAccess<'de> for Variant<'de> {
}
}
}
+
+pub mod ordered_object {
+
+ use crate::json::Val;
+ use serde::de::{MapAccess, Visitor};
+ use serde::{Deserializer, Serializer};
+ use std::fmt;
+
+ pub fn serialize(pairs: &Vec<(String, Val)>, serializer: S) -> Result
+ where
+ S: Serializer,
+ {
+ use serde::ser::SerializeMap;
+ let mut map = serializer.serialize_map(Some(pairs.len()))?;
+ for (k, v) in pairs {
+ if let Val::Removed = v {
+ continue;
+ }
+ map.serialize_entry(k, v)?;
+ }
+ map.end()
+ }
+
+ pub fn deserialize<'de, D>(deserializer: D) -> Result, D::Error>
+ where
+ D: Deserializer<'de>,
+ {
+ struct OrderedMapVisitor;
+
+ impl<'de> Visitor<'de> for OrderedMapVisitor {
+ type Value = Vec<(String, Val)>;
+
+ fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+ formatter.write_str("a map")
+ }
+
+ fn visit_map(self, mut access: M) -> Result
+ where
+ M: MapAccess<'de>,
+ {
+ let mut pairs = match access.size_hint() {
+ Some(size) => Vec::with_capacity(size),
+ None => Vec::new(),
+ };
+
+ while let Some((key, value)) = access.next_entry()? {
+ pairs.push((key, value));
+ }
+
+ Ok(pairs)
+ }
+ }
+
+ deserializer.deserialize_map(OrderedMapVisitor)
+ }
+}
diff --git a/core/json/error.rs b/core/json/error.rs
index 870601738..08d17e309 100644
--- a/core/json/error.rs
+++ b/core/json/error.rs
@@ -106,3 +106,11 @@ pub fn set_location(res: &mut Result, span: &Span<'_>) {
}
}
}
+
+impl From for crate::LimboError {
+ fn from(err: Error) -> Self {
+ match err {
+ Error::Message { msg, .. } => crate::LimboError::ParseError(msg),
+ }
+ }
+}
diff --git a/core/json/json_operations.rs b/core/json/json_operations.rs
new file mode 100644
index 000000000..086090fea
--- /dev/null
+++ b/core/json/json_operations.rs
@@ -0,0 +1,594 @@
+use std::collections::VecDeque;
+
+use crate::{
+ json::{mutate_json_by_path, Target},
+ types::OwnedValue,
+};
+
+use super::{convert_json_to_db_type, get_json_value, json_path::json_path, Val};
+
+/// Represents a single patch operation in the merge queue.
+///
+/// Used internally by the `merge_patch` function to track the path and value
+/// for each pending merge operation.
+#[derive(Debug, Clone)]
+struct PatchOperation {
+ path_start: usize,
+ path_len: usize,
+ patch: Val,
+}
+
+/// The function follows RFC 7386 JSON Merge Patch semantics:
+/// * If the patch is null, the target is replaced with null
+/// * If the patch contains a scalar value, the target is replaced with that value
+/// * If both target and patch are objects, the patch is recursively applied
+/// * null values in the patch result in property removal from the target
+pub fn json_patch(target: &OwnedValue, patch: &OwnedValue) -> crate::Result {
+ match (target, patch) {
+ (OwnedValue::Blob(_), _) | (_, OwnedValue::Blob(_)) => {
+ crate::bail_constraint_error!("blob is not supported!");
+ }
+ _ => (),
+ }
+
+ let mut parsed_target = get_json_value(target)?;
+ let parsed_patch = get_json_value(patch)?;
+ let mut patcher = JsonPatcher::new(16);
+ patcher.apply_patch(&mut parsed_target, parsed_patch);
+
+ convert_json_to_db_type(&parsed_target, false)
+}
+
+#[derive(Debug)]
+struct JsonPatcher {
+ queue: VecDeque,
+ path_storage: Vec,
+}
+
+impl JsonPatcher {
+ fn new(queue_capacity: usize) -> Self {
+ Self {
+ queue: VecDeque::with_capacity(queue_capacity),
+ path_storage: Vec::with_capacity(256),
+ }
+ }
+
+ fn apply_patch(&mut self, target: &mut Val, patch: Val) {
+ self.queue.push_back(PatchOperation {
+ path_start: 0,
+ path_len: 0,
+ patch,
+ });
+
+ while let Some(op) = self.queue.pop_front() {
+ if let Some(current) = self.navigate_to_target(target, op.path_start, op.path_len) {
+ self.apply_operation(current, op);
+ } else {
+ continue;
+ }
+ }
+ }
+
+ fn navigate_to_target<'a>(
+ &self,
+ target: &'a mut Val,
+ path_start: usize,
+ path_len: usize,
+ ) -> Option<&'a mut Val> {
+ let mut current = target;
+ for i in 0..path_len {
+ let key = self.path_storage[path_start + i];
+ if let Val::Object(ref mut obj) = current {
+ current = &mut obj
+ .get_mut(key)
+ .unwrap_or_else(|| {
+ panic!("Invalid path at depth {}: key '{}' not found", i, key)
+ })
+ .1;
+ } else {
+ return None;
+ }
+ }
+ Some(current)
+ }
+
+ fn apply_operation(&mut self, current: &mut Val, operation: PatchOperation) {
+ let path_start = operation.path_start;
+ let path_len = operation.path_len;
+ match (current, operation.patch) {
+ (current_val, Val::Null) => *current_val = Val::Removed,
+ (Val::Object(target_map), Val::Object(patch_map)) => {
+ self.merge_objects(target_map, patch_map, path_start, path_len);
+ }
+ (current_val, patch_val) => *current_val = patch_val,
+ }
+ }
+
+ fn merge_objects(
+ &mut self,
+ target_map: &mut Vec<(String, Val)>,
+ patch_map: Vec<(String, Val)>,
+ path_start: usize,
+ path_len: usize,
+ ) {
+ for (key, patch_val) in patch_map {
+ self.process_key_value(target_map, key, patch_val, path_start, path_len);
+ }
+ }
+
+ fn process_key_value(
+ &mut self,
+ target_map: &mut Vec<(String, Val)>,
+ key: String,
+ patch_val: Val,
+ path_start: usize,
+ path_len: usize,
+ ) {
+ if let Some(pos) = target_map
+ .iter()
+ .position(|(target_key, _)| target_key == &key)
+ {
+ self.queue_nested_patch(pos, patch_val, path_start, path_len);
+ } else if !matches!(patch_val, Val::Null) {
+ target_map.push((key, Val::Object(vec![])));
+ self.queue_nested_patch(target_map.len() - 1, patch_val, path_start, path_len)
+ }
+ }
+
+ fn queue_nested_patch(&mut self, pos: usize, val: Val, path_start: usize, path_len: usize) {
+ let new_path_start = self.path_storage.len();
+ let new_path_len = path_len + 1;
+ for i in 0..path_len {
+ self.path_storage.push(self.path_storage[path_start + i]);
+ }
+ self.path_storage.push(pos);
+ self.queue.push_back(PatchOperation {
+ path_start: new_path_start,
+ path_len: new_path_len,
+ patch: val,
+ });
+ }
+}
+
+pub fn json_remove(args: &[OwnedValue]) -> crate::Result {
+ if args.is_empty() {
+ return Ok(OwnedValue::Null);
+ }
+
+ let mut parsed_target = get_json_value(&args[0])?;
+ if args.len() == 1 {
+ return Ok(args[0].clone());
+ }
+
+ let paths: Result, _> = args[1..]
+ .iter()
+ .map(|path| {
+ if let OwnedValue::Text(path) = path {
+ json_path(&path.value)
+ } else {
+ crate::bail_constraint_error!("bad JSON path: {:?}", path.to_string())
+ }
+ })
+ .collect();
+ let paths = paths?;
+
+ for path in paths {
+ mutate_json_by_path(&mut parsed_target, path, |val| match val {
+ Target::Array(arr, index) => {
+ arr.remove(index);
+ }
+ Target::Value(val) => *val = Val::Removed,
+ });
+ }
+
+ convert_json_to_db_type(&parsed_target, false)
+}
+
+#[cfg(test)]
+mod tests {
+ use std::rc::Rc;
+
+ use crate::types::LimboText;
+
+ use super::*;
+
+ fn create_text(s: &str) -> OwnedValue {
+ OwnedValue::Text(LimboText::new(Rc::new(s.to_string())))
+ }
+
+ fn create_json(s: &str) -> OwnedValue {
+ OwnedValue::Text(LimboText::json(Rc::new(s.to_string())))
+ }
+
+ #[test]
+ fn test_new_patcher() {
+ let patcher = JsonPatcher::new(10);
+ assert_eq!(patcher.queue.capacity(), 10);
+ assert_eq!(patcher.path_storage.capacity(), 256);
+ assert!(patcher.queue.is_empty());
+ assert!(patcher.path_storage.is_empty());
+ }
+
+ #[test]
+ fn test_simple_value_replacement() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![("key1".to_string(), Val::String("old".to_string()))]);
+ let patch = Val::Object(vec![("key1".to_string(), Val::String("new".to_string()))]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ assert_eq!(map[0].1, Val::String("new".to_string()));
+ } else {
+ panic!("Expected object");
+ }
+ }
+
+ #[test]
+ fn test_nested_object_patch() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![(
+ "a".to_string(),
+ Val::Object(vec![("b".to_string(), Val::String("old".to_string()))]),
+ )]);
+ let patch = Val::Object(vec![(
+ "a".to_string(),
+ Val::Object(vec![("b".to_string(), Val::String("new".to_string()))]),
+ )]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ if let Val::Object(nested) = &map[0].1 {
+ assert_eq!(nested[0].1, Val::String("new".to_string()));
+ } else {
+ panic!("Expected nested object");
+ }
+ }
+ }
+
+ #[test]
+ fn test_null_removal() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![
+ ("keep".to_string(), Val::String("value".to_string())),
+ ("remove".to_string(), Val::String("value".to_string())),
+ ]);
+ let patch = Val::Object(vec![("remove".to_string(), Val::Null)]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ assert_eq!(map[0].1, Val::String("value".to_string()));
+ assert_eq!(map[1].1, Val::Removed);
+ }
+ }
+
+ #[test]
+ fn test_duplicate_keys_first_occurrence() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![
+ ("key".to_string(), Val::String("first".to_string())),
+ ("key".to_string(), Val::String("second".to_string())),
+ ("key".to_string(), Val::String("third".to_string())),
+ ]);
+ let patch = Val::Object(vec![(
+ "key".to_string(),
+ Val::String("modified".to_string()),
+ )]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ assert_eq!(map[0].1, Val::String("modified".to_string()));
+ assert_eq!(map[1].1, Val::String("second".to_string()));
+ assert_eq!(map[2].1, Val::String("third".to_string()));
+ }
+ }
+
+ #[test]
+ fn test_add_new_key() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![(
+ "existing".to_string(),
+ Val::String("value".to_string()),
+ )]);
+ let patch = Val::Object(vec![("new".to_string(), Val::String("value".to_string()))]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ assert_eq!(map.len(), 2);
+ assert_eq!(map[1].0, "new");
+ assert_eq!(map[1].1, Val::String("value".to_string()));
+ }
+ }
+
+ #[test]
+ fn test_deep_nested_patch() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![(
+ "level1".to_string(),
+ Val::Object(vec![(
+ "level2".to_string(),
+ Val::Object(vec![("level3".to_string(), Val::String("old".to_string()))]),
+ )]),
+ )]);
+ let patch = Val::Object(vec![(
+ "level1".to_string(),
+ Val::Object(vec![(
+ "level2".to_string(),
+ Val::Object(vec![("level3".to_string(), Val::String("new".to_string()))]),
+ )]),
+ )]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(l1) = target {
+ if let Val::Object(l2) = &l1[0].1 {
+ if let Val::Object(l3) = &l2[0].1 {
+ assert_eq!(l3[0].1, Val::String("new".to_string()));
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn test_null_patch_on_nonexistent_key() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![(
+ "existing".to_string(),
+ Val::String("value".to_string()),
+ )]);
+ let patch = Val::Object(vec![("nonexistent".to_string(), Val::Null)]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ assert_eq!(map.len(), 1); // Should not add new key for null patch
+ assert_eq!(map[0].0, "existing");
+ }
+ }
+
+ #[test]
+ fn test_nested_duplicate_keys() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![(
+ "outer".to_string(),
+ Val::Object(vec![
+ ("inner".to_string(), Val::String("first".to_string())),
+ ("inner".to_string(), Val::String("second".to_string())),
+ ]),
+ )]);
+ let patch = Val::Object(vec![(
+ "outer".to_string(),
+ Val::Object(vec![(
+ "inner".to_string(),
+ Val::String("modified".to_string()),
+ )]),
+ )]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(outer) = target {
+ if let Val::Object(inner) = &outer[0].1 {
+ assert_eq!(inner[0].1, Val::String("modified".to_string()));
+ assert_eq!(inner[1].1, Val::String("second".to_string()));
+ }
+ }
+ }
+
+ #[test]
+ #[should_panic(expected = "Invalid path")]
+ fn test_invalid_path_navigation() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![("a".to_string(), Val::Object(vec![]))]);
+ patcher.path_storage.push(0);
+ patcher.path_storage.push(999); // Invalid index
+
+ patcher.navigate_to_target(&mut target, 0, 2);
+ }
+
+ #[test]
+ fn test_merge_empty_objects() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![]);
+ let patch = Val::Object(vec![]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ if let Val::Object(map) = target {
+ assert!(map.is_empty());
+ }
+ }
+
+ #[test]
+ fn test_path_storage_growth() {
+ let mut patcher = JsonPatcher::new(10);
+ let mut target = Val::Object(vec![(
+ "a".to_string(),
+ Val::Object(vec![("b".to_string(), Val::Object(vec![]))]),
+ )]);
+ let patch = Val::Object(vec![(
+ "a".to_string(),
+ Val::Object(vec![("b".to_string(), Val::String("value".to_string()))]),
+ )]);
+
+ patcher.apply_patch(&mut target, patch);
+
+ // Path storage should contain [0, 0] for accessing a.b
+ assert_eq!(patcher.path_storage.len(), 3);
+ assert_eq!(patcher.path_storage[0], 0);
+ assert_eq!(patcher.path_storage[1], 0);
+ }
+
+ #[test]
+ fn test_basic_text_replacement() {
+ let target = create_text(r#"{"name":"John","age":"30"}"#);
+ let patch = create_text(r#"{"age":"31"}"#);
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(result, create_json(r#"{"name":"John","age":"31"}"#));
+ }
+
+ #[test]
+ fn test_null_field_removal() {
+ let target = create_text(r#"{"name":"John","email":"john@example.com"}"#);
+ let patch = create_text(r#"{"email":null}"#);
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(result, create_json(r#"{"name":"John"}"#));
+ }
+
+ #[test]
+ fn test_nested_object_merge() {
+ let target =
+ create_text(r#"{"user":{"name":"John","details":{"age":"30","score":"95.5"}}}"#);
+
+ let patch = create_text(r#"{"user":{"details":{"score":"97.5"}}}"#);
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(
+ result,
+ create_json(r#"{"user":{"name":"John","details":{"age":"30","score":"97.5"}}}"#)
+ );
+ }
+
+ #[test]
+ #[should_panic(expected = "blob is not supported!")]
+ fn test_blob_not_supported() {
+ let target = OwnedValue::Blob(Rc::new(vec![1, 2, 3]));
+ let patch = create_text("{}");
+ json_patch(&target, &patch).unwrap();
+ }
+
+ #[test]
+ fn test_deep_null_replacement() {
+ let target = create_text(r#"{"level1":{"level2":{"keep":"value","remove":"value"}}}"#);
+
+ let patch = create_text(r#"{"level1":{"level2":{"remove":null}}}"#);
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(
+ result,
+ create_json(r#"{"level1":{"level2":{"keep":"value"}}}"#)
+ );
+ }
+
+ #[test]
+ fn test_empty_patch() {
+ let target = create_json(r#"{"name":"John","age":"30"}"#);
+ let patch = create_text("{}");
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(result, target);
+ }
+
+ #[test]
+ fn test_add_new_field() {
+ let target = create_text(r#"{"existing":"value"}"#);
+ let patch = create_text(r#"{"new":"field"}"#);
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(result, create_json(r#"{"existing":"value","new":"field"}"#));
+ }
+
+ #[test]
+ fn test_complete_object_replacement() {
+ let target = create_text(r#"{"old":{"nested":"value"}}"#);
+ let patch = create_text(r#"{"old":"new_value"}"#);
+
+ let result = json_patch(&target, &patch).unwrap();
+ assert_eq!(result, create_json(r#"{"old":"new_value"}"#));
+ }
+
+ #[test]
+ fn test_json_remove_empty_args() {
+ let args = vec![];
+ assert_eq!(json_remove(&args).unwrap(), OwnedValue::Null);
+ }
+
+ #[test]
+ fn test_json_remove_array_element() {
+ let args = vec![create_json(r#"[1,2,3,4,5]"#), create_text("$[2]")];
+
+ let result = json_remove(&args).unwrap();
+ match result {
+ OwnedValue::Text(t) => assert_eq!(t.value.as_str(), "[1,2,4,5]"),
+ _ => panic!("Expected Text value"),
+ }
+ }
+
+ #[test]
+ fn test_json_remove_multiple_paths() {
+ let args = vec![
+ create_json(r#"{"a": 1, "b": 2, "c": 3}"#),
+ create_text("$.a"),
+ create_text("$.c"),
+ ];
+
+ let result = json_remove(&args).unwrap();
+ match result {
+ OwnedValue::Text(t) => assert_eq!(t.value.as_str(), r#"{"b":2}"#),
+ _ => panic!("Expected Text value"),
+ }
+ }
+
+ #[test]
+ fn test_json_remove_nested_paths() {
+ let args = vec![
+ create_json(r#"{"a": {"b": {"c": 1, "d": 2}}}"#),
+ create_text("$.a.b.c"),
+ ];
+
+ let result = json_remove(&args).unwrap();
+ match result {
+ OwnedValue::Text(t) => assert_eq!(t.value.as_str(), r#"{"a":{"b":{"d":2}}}"#),
+ _ => panic!("Expected Text value"),
+ }
+ }
+
+ #[test]
+ fn test_json_remove_duplicate_keys() {
+ let args = vec![
+ create_json(r#"{"a": 1, "a": 2, "a": 3}"#),
+ create_text("$.a"),
+ ];
+
+ let result = json_remove(&args).unwrap();
+ match result {
+ OwnedValue::Text(t) => assert_eq!(t.value.as_str(), r#"{"a":2,"a":3}"#),
+ _ => panic!("Expected Text value"),
+ }
+ }
+
+ #[test]
+ fn test_json_remove_invalid_path() {
+ let args = vec![
+ create_json(r#"{"a": 1}"#),
+ OwnedValue::Integer(42), // Invalid path type
+ ];
+
+ assert!(json_remove(&args).is_err());
+ }
+
+ #[test]
+ fn test_json_remove_complex_case() {
+ let args = vec![
+ create_json(r#"{"a":[1,2,3],"b":{"x":1,"x":2},"c":[{"y":1},{"y":2}]}"#),
+ create_text("$.a[1]"),
+ create_text("$.b.x"),
+ create_text("$.c[0].y"),
+ ];
+
+ let result = json_remove(&args).unwrap();
+ match result {
+ OwnedValue::Text(t) => {
+ let value = t.value.as_str();
+ assert!(value.contains(r#"[1,3]"#));
+ assert!(value.contains(r#"{"x":2}"#));
+ }
+ _ => panic!("Expected Text value"),
+ }
+ }
+}
diff --git a/core/json/json_path.pest b/core/json/json_path.pest
index 71a462edc..590a3df23 100644
--- a/core/json/json_path.pest
+++ b/core/json/json_path.pest
@@ -4,5 +4,5 @@ array_locator = ${ "[" ~ negative_index_indicator? ~ array_offset ~ "]" }
relaxed_array_locator = ${ negative_index_indicator? ~ array_offset }
root = ${ "$" }
-json_path_key = ${ identifier | string }
+json_path_key = ${ identifier | string | ASCII_DIGIT+ }
path = ${ SOI ~ root ~ (array_locator | "." ~ json_path_key)* ~ EOI }
diff --git a/core/json/mod.rs b/core/json/mod.rs
index 10e682148..13c2bf98d 100644
--- a/core/json/mod.rs
+++ b/core/json/mod.rs
@@ -1,17 +1,21 @@
mod de;
mod error;
+mod json_operations;
mod json_path;
mod ser;
use std::rc::Rc;
pub use crate::json::de::from_str;
+use crate::json::de::ordered_object;
use crate::json::error::Error as JsonError;
+pub use crate::json::json_operations::{json_patch, json_remove};
use crate::json::json_path::{json_path, JsonPath, PathElement};
pub use crate::json::ser::to_string;
use crate::types::{LimboText, OwnedValue, TextSubtype};
use indexmap::IndexMap;
use jsonb::Error as JsonbError;
+use ser::to_string_pretty;
use serde::{Deserialize, Serialize};
#[derive(Serialize, Deserialize, Debug, PartialEq, Clone)]
@@ -23,10 +27,12 @@ pub enum Val {
Float(f64),
String(String),
Array(Vec),
- Object(IndexMap),
+ Removed,
+ #[serde(with = "ordered_object")]
+ Object(Vec<(String, Val)>),
}
-pub fn get_json(json_value: &OwnedValue) -> crate::Result {
+pub fn get_json(json_value: &OwnedValue, indent: Option<&str>) -> crate::Result {
match json_value {
OwnedValue::Text(ref t) => {
// optimization: once we know the subtype is a valid JSON, we do not have
@@ -36,7 +42,10 @@ pub fn get_json(json_value: &OwnedValue) -> crate::Result {
}
let json_val = get_json_value(json_value)?;
- let json = to_string(&json_val).unwrap();
+ let json = match indent {
+ Some(indent) => to_string_pretty(&json_val, indent)?,
+ None => to_string(&json_val)?,
+ };
Ok(OwnedValue::Text(LimboText::json(Rc::new(json))))
}
@@ -52,7 +61,10 @@ pub fn get_json(json_value: &OwnedValue) -> crate::Result {
OwnedValue::Null => Ok(OwnedValue::Null),
_ => {
let json_val = get_json_value(json_value)?;
- let json = to_string(&json_val).unwrap();
+ let json = match indent {
+ Some(indent) => to_string_pretty(&json_val, indent)?,
+ None => to_string(&json_val)?,
+ };
Ok(OwnedValue::Text(LimboText::json(Rc::new(json))))
}
@@ -143,6 +155,45 @@ pub fn json_array_length(
}
}
+pub fn json_set(json: &OwnedValue, values: &[OwnedValue]) -> crate::Result {
+ let mut json_value = get_json_value(json)?;
+
+ values
+ .chunks(2)
+ .map(|chunk| match chunk {
+ [path, value] => {
+ let path = json_path_from_owned_value(path, true)?;
+
+ if let Some(path) = path {
+ let new_value = match value {
+ OwnedValue::Text(LimboText {
+ value,
+ subtype: TextSubtype::Text,
+ }) => Val::String(value.to_string()),
+ _ => get_json_value(value)?,
+ };
+
+ let mut new_json_value = json_value.clone();
+
+ match create_and_mutate_json_by_path(&mut new_json_value, path, |val| match val
+ {
+ Target::Array(arr, index) => arr[index] = new_value.clone(),
+ Target::Value(val) => *val = new_value.clone(),
+ }) {
+ Some(_) => json_value = new_json_value,
+ _ => {}
+ }
+ }
+
+ Ok(())
+ }
+ _ => crate::bail_constraint_error!("json_set needs an odd number of arguments"),
+ })
+ .collect::>()?;
+
+ convert_json_to_db_type(&json_value, true)
+}
+
/// Implements the -> operator. Always returns a proper JSON value.
/// https://sqlite.org/json1.html#the_and_operators
pub fn json_arrow_extract(value: &OwnedValue, path: &OwnedValue) -> crate::Result {
@@ -154,7 +205,7 @@ pub fn json_arrow_extract(value: &OwnedValue, path: &OwnedValue) -> crate::Resul
let extracted = json_extract_single(&json, path, false)?;
if let Some(val) = extracted {
- let json = to_string(val).unwrap();
+ let json = to_string(val)?;
Ok(OwnedValue::Text(LimboText::json(Rc::new(json))))
} else {
@@ -211,7 +262,7 @@ pub fn json_extract(value: &OwnedValue, paths: &[OwnedValue]) -> crate::Result crate::Result crate::Result {
match extracted {
+ Val::Removed => Ok(OwnedValue::Null),
Val::Null => Ok(OwnedValue::Null),
Val::Float(f) => Ok(OwnedValue::Float(*f)),
Val::Integer(i) => Ok(OwnedValue::Integer(*i)),
@@ -248,7 +300,7 @@ fn convert_json_to_db_type(extracted: &Val, all_as_db: bool) -> crate::Result Ok(OwnedValue::Text(LimboText::new(Rc::new(s.clone())))),
_ => {
- let json = to_string(&extracted).unwrap();
+ let json = to_string(&extracted)?;
if all_as_db {
Ok(OwnedValue::Text(LimboText::new(Rc::new(json))))
} else {
@@ -311,6 +363,7 @@ pub fn json_type(value: &OwnedValue, path: Option<&OwnedValue>) -> crate::Result
Val::String(_) => "text",
Val::Array(_) => "array",
Val::Object(_) => "object",
+ Val::Removed => unreachable!(),
};
Ok(OwnedValue::Text(LimboText::json(Rc::new(val.to_string()))))
@@ -327,32 +380,9 @@ fn json_extract_single<'a>(
path: &OwnedValue,
strict: bool,
) -> crate::Result> {
- let json_path = if strict {
- match path {
- OwnedValue::Text(t) => json_path(t.value.as_str())?,
- OwnedValue::Null => return Ok(None),
- _ => crate::bail_constraint_error!("JSON path error near: {:?}", path.to_string()),
- }
- } else {
- match path {
- OwnedValue::Text(t) => {
- if t.value.starts_with("$") {
- json_path(t.value.as_str())?
- } else {
- JsonPath {
- elements: vec![PathElement::Root(), PathElement::Key(t.value.to_string())],
- }
- }
- }
- OwnedValue::Null => return Ok(None),
- OwnedValue::Integer(i) => JsonPath {
- elements: vec![PathElement::Root(), PathElement::ArrayLocator(*i as i32)],
- },
- OwnedValue::Float(f) => JsonPath {
- elements: vec![PathElement::Root(), PathElement::Key(f.to_string())],
- },
- _ => crate::bail_constraint_error!("JSON path error near: {:?}", path.to_string()),
- }
+ let json_path = match json_path_from_owned_value(path, strict)? {
+ Some(path) => path,
+ None => return Ok(None),
};
let mut current_element = &Val::Null;
@@ -367,7 +397,7 @@ fn json_extract_single<'a>(
match current_element {
Val::Object(map) => {
- if let Some(value) = map.get(key) {
+ if let Some((_, value)) = map.iter().find(|(k, _)| k == key) {
current_element = value;
} else {
return Ok(None);
@@ -398,6 +428,182 @@ fn json_extract_single<'a>(
Ok(Some(current_element))
}
+fn json_path_from_owned_value(path: &OwnedValue, strict: bool) -> crate::Result> {
+ let json_path = if strict {
+ match path {
+ OwnedValue::Text(t) => json_path(t.value.as_str())?,
+ OwnedValue::Null => return Ok(None),
+ _ => crate::bail_constraint_error!("JSON path error near: {:?}", path.to_string()),
+ }
+ } else {
+ match path {
+ OwnedValue::Text(t) => {
+ if t.value.starts_with("$") {
+ json_path(t.value.as_str())?
+ } else {
+ JsonPath {
+ elements: vec![PathElement::Root(), PathElement::Key(t.value.to_string())],
+ }
+ }
+ }
+ OwnedValue::Null => return Ok(None),
+ OwnedValue::Integer(i) => JsonPath {
+ elements: vec![PathElement::Root(), PathElement::ArrayLocator(*i as i32)],
+ },
+ OwnedValue::Float(f) => JsonPath {
+ elements: vec![PathElement::Root(), PathElement::Key(f.to_string())],
+ },
+ _ => crate::bail_constraint_error!("JSON path error near: {:?}", path.to_string()),
+ }
+ };
+
+ Ok(Some(json_path))
+}
+
+enum Target<'a> {
+ Array(&'a mut Vec, usize),
+ Value(&'a mut Val),
+}
+
+fn mutate_json_by_path(json: &mut Val, path: JsonPath, closure: F) -> Option
+where
+ F: FnMut(Target) -> R,
+{
+ find_target(json, &path).map(closure)
+}
+
+fn find_target<'a>(json: &'a mut Val, path: &JsonPath) -> Option> {
+ let mut current = json;
+ for (i, key) in path.elements.iter().enumerate() {
+ let is_last = i == path.elements.len() - 1;
+ match key {
+ PathElement::Root() => continue,
+ PathElement::ArrayLocator(index) => match current {
+ Val::Array(arr) => {
+ if let Some(index) = match index {
+ i if *i < 0 => arr.len().checked_sub(i.unsigned_abs() as usize),
+ i => ((*i as usize) < arr.len()).then_some(*i as usize),
+ } {
+ if is_last {
+ return Some(Target::Array(arr, index));
+ } else {
+ current = &mut arr[index];
+ }
+ } else {
+ return None;
+ }
+ }
+ _ => {
+ return None;
+ }
+ },
+ PathElement::Key(key) => match current {
+ Val::Object(obj) => {
+ if let Some(pos) = &obj
+ .iter()
+ .position(|(k, v)| k == key && !matches!(v, Val::Removed))
+ {
+ let val = &mut obj[*pos].1;
+ current = val;
+ } else {
+ return None;
+ }
+ }
+ _ => {
+ return None;
+ }
+ },
+ }
+ }
+ Some(Target::Value(current))
+}
+
+fn create_and_mutate_json_by_path(json: &mut Val, path: JsonPath, closure: F) -> Option
+where
+ F: FnOnce(Target) -> R,
+{
+ find_or_create_target(json, &path).map(closure)
+}
+
+fn find_or_create_target<'a>(json: &'a mut Val, path: &JsonPath) -> Option> {
+ let mut current = json;
+ for (i, key) in path.elements.iter().enumerate() {
+ let is_last = i == path.elements.len() - 1;
+ match key {
+ PathElement::Root() => continue,
+ PathElement::ArrayLocator(index) => match current {
+ Val::Array(arr) => {
+ if let Some(index) = match index {
+ i if *i < 0 => arr.len().checked_sub(i.unsigned_abs() as usize),
+ i => Some(*i as usize),
+ } {
+ if is_last {
+ if index == arr.len() {
+ arr.push(Val::Null);
+ }
+
+ if index >= arr.len() {
+ return None;
+ }
+
+ return Some(Target::Array(arr, index));
+ } else {
+ if index == arr.len() {
+ arr.push(
+ if matches!(path.elements[i + 1], PathElement::ArrayLocator(_))
+ {
+ Val::Array(vec![])
+ } else {
+ Val::Object(vec![])
+ },
+ );
+ }
+
+ if index >= arr.len() {
+ return None;
+ }
+
+ current = &mut arr[index];
+ }
+ } else {
+ return None;
+ }
+ }
+ _ => {
+ *current = Val::Array(vec![]);
+ }
+ },
+ PathElement::Key(key) => match current {
+ Val::Object(obj) => {
+ if let Some(pos) = &obj
+ .iter()
+ .position(|(k, v)| k == key && !matches!(v, Val::Removed))
+ {
+ let val = &mut obj[*pos].1;
+ current = val;
+ } else {
+ let element = if !is_last
+ && matches!(path.elements[i + 1], PathElement::ArrayLocator(_))
+ {
+ Val::Array(vec![])
+ } else {
+ Val::Object(vec![])
+ };
+
+ obj.push((key.clone(), element));
+ let index = obj.len() - 1;
+ current = &mut obj[index].1;
+ }
+ }
+ _ => {
+ return None;
+ }
+ },
+ }
+ }
+ Some(Target::Value(current))
+}
+
pub fn json_error_position(json: &OwnedValue) -> crate::Result {
match json {
OwnedValue::Text(t) => match from_str::(&t.value) {
@@ -444,10 +650,25 @@ pub fn json_object(values: &[OwnedValue]) -> crate::Result {
})
.collect::, _>>()?;
- let result = crate::json::to_string(&value_map).unwrap();
+ let result = crate::json::to_string(&value_map)?;
Ok(OwnedValue::Text(LimboText::json(Rc::new(result))))
}
+pub fn is_json_valid(json_value: &OwnedValue) -> crate::Result {
+ match json_value {
+ OwnedValue::Text(ref t) => match from_str::(&t.value) {
+ Ok(_) => Ok(OwnedValue::Integer(1)),
+ Err(_) => Ok(OwnedValue::Integer(0)),
+ },
+ OwnedValue::Blob(b) => match jsonb::from_slice(b) {
+ Ok(_) => Ok(OwnedValue::Integer(1)),
+ Err(_) => Ok(OwnedValue::Integer(0)),
+ },
+ OwnedValue::Null => Ok(OwnedValue::Null),
+ _ => Ok(OwnedValue::Integer(1)),
+ }
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -456,7 +677,7 @@ mod tests {
#[test]
fn test_get_json_valid_json5() {
let input = OwnedValue::build_text(Rc::new("{ key: 'value' }".to_string()));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"key\":\"value\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -468,7 +689,7 @@ mod tests {
#[test]
fn test_get_json_valid_json5_double_single_quotes() {
let input = OwnedValue::build_text(Rc::new("{ key: ''value'' }".to_string()));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"key\":\"value\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -480,7 +701,7 @@ mod tests {
#[test]
fn test_get_json_valid_json5_infinity() {
let input = OwnedValue::build_text(Rc::new("{ \"key\": Infinity }".to_string()));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("{\"key\":9e999}"));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -492,7 +713,7 @@ mod tests {
#[test]
fn test_get_json_valid_json5_negative_infinity() {
let input = OwnedValue::build_text(Rc::new("{ \"key\": -Infinity }".to_string()));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("{\"key\":-9e999}"));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -504,7 +725,7 @@ mod tests {
#[test]
fn test_get_json_valid_json5_nan() {
let input = OwnedValue::build_text(Rc::new("{ \"key\": NaN }".to_string()));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("{\"key\":null}"));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -516,7 +737,7 @@ mod tests {
#[test]
fn test_get_json_invalid_json5() {
let input = OwnedValue::build_text(Rc::new("{ key: value }".to_string()));
- let result = get_json(&input);
+ let result = get_json(&input, None);
match result {
Ok(_) => panic!("Expected error for malformed JSON"),
Err(e) => assert!(e.to_string().contains("malformed JSON")),
@@ -526,7 +747,7 @@ mod tests {
#[test]
fn test_get_json_valid_jsonb() {
let input = OwnedValue::build_text(Rc::new("{\"key\":\"value\"}".to_string()));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"key\":\"value\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -538,7 +759,7 @@ mod tests {
#[test]
fn test_get_json_invalid_jsonb() {
let input = OwnedValue::build_text(Rc::new("{key:\"value\"".to_string()));
- let result = get_json(&input);
+ let result = get_json(&input, None);
match result {
Ok(_) => panic!("Expected error for malformed JSON"),
Err(e) => assert!(e.to_string().contains("malformed JSON")),
@@ -549,7 +770,7 @@ mod tests {
fn test_get_json_blob_valid_jsonb() {
let binary_json = b"\x40\0\0\x01\x10\0\0\x03\x10\0\0\x03\x61\x73\x64\x61\x64\x66".to_vec();
let input = OwnedValue::Blob(Rc::new(binary_json));
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Text(result_str) = result {
assert!(result_str.value.contains("\"asd\":\"adf\""));
assert_eq!(result_str.subtype, TextSubtype::Json);
@@ -562,7 +783,7 @@ mod tests {
fn test_get_json_blob_invalid_jsonb() {
let binary_json: Vec = vec![0xA2, 0x62, 0x6B, 0x31, 0x62, 0x76]; // Incomplete binary JSON
let input = OwnedValue::Blob(Rc::new(binary_json));
- let result = get_json(&input);
+ let result = get_json(&input, None);
match result {
Ok(_) => panic!("Expected error for malformed JSON"),
Err(e) => assert!(e.to_string().contains("malformed JSON")),
@@ -572,7 +793,7 @@ mod tests {
#[test]
fn test_get_json_non_text() {
let input = OwnedValue::Null;
- let result = get_json(&input).unwrap();
+ let result = get_json(&input, None).unwrap();
if let OwnedValue::Null = result {
// Test passed
} else {
@@ -729,7 +950,7 @@ mod tests {
#[test]
fn test_json_array_length_simple_json_subtype() {
let input = OwnedValue::build_text(Rc::new("[1,2,3]".to_string()));
- let wrapped = get_json(&input).unwrap();
+ let wrapped = get_json(&input, None).unwrap();
let result = json_array_length(&wrapped, None).unwrap();
if let OwnedValue::Integer(res) = result {
@@ -983,18 +1204,484 @@ mod tests {
.contains("json_object requires an even number of values")),
}
}
-}
-pub fn is_json_valid(json_value: &OwnedValue) -> crate::Result {
- match json_value {
- OwnedValue::Text(ref t) => match from_str::(&t.value) {
- Ok(_) => Ok(OwnedValue::Integer(1)),
- Err(_) => Ok(OwnedValue::Integer(0)),
- },
- OwnedValue::Blob(b) => match jsonb::from_slice(b) {
- Ok(_) => Ok(OwnedValue::Integer(1)),
- Err(_) => Ok(OwnedValue::Integer(0)),
- },
- OwnedValue::Null => Ok(OwnedValue::Null),
- _ => Ok(OwnedValue::Integer(1)),
+
+ #[test]
+ fn test_find_target_array() {
+ let mut val = Val::Array(vec![
+ Val::String("first".to_string()),
+ Val::String("second".to_string()),
+ ]);
+ let path = JsonPath {
+ elements: vec![PathElement::ArrayLocator(0)],
+ };
+
+ match find_target(&mut val, &path) {
+ Some(Target::Array(_, idx)) => assert_eq!(idx, 0),
+ _ => panic!("Expected Array target"),
+ }
+ }
+
+ #[test]
+ fn test_find_target_negative_index() {
+ let mut val = Val::Array(vec![
+ Val::String("first".to_string()),
+ Val::String("second".to_string()),
+ ]);
+ let path = JsonPath {
+ elements: vec![PathElement::ArrayLocator(-1)],
+ };
+
+ match find_target(&mut val, &path) {
+ Some(Target::Array(_, idx)) => assert_eq!(idx, 1),
+ _ => panic!("Expected Array target"),
+ }
+ }
+
+ #[test]
+ fn test_find_target_object() {
+ let mut val = Val::Object(vec![("key".to_string(), Val::String("value".to_string()))]);
+ let path = JsonPath {
+ elements: vec![PathElement::Key("key".to_string())],
+ };
+
+ match find_target(&mut val, &path) {
+ Some(Target::Value(_)) => {}
+ _ => panic!("Expected Value target"),
+ }
+ }
+
+ #[test]
+ fn test_find_target_removed() {
+ let mut val = Val::Object(vec![
+ ("key".to_string(), Val::Removed),
+ ("key".to_string(), Val::String("value".to_string())),
+ ]);
+ let path = JsonPath {
+ elements: vec![PathElement::Key("key".to_string())],
+ };
+
+ match find_target(&mut val, &path) {
+ Some(Target::Value(val)) => assert!(matches!(val, Val::String(_))),
+ _ => panic!("Expected second value, not removed"),
+ }
+ }
+
+ #[test]
+ fn test_mutate_json() {
+ let mut val = Val::Array(vec![Val::String("test".to_string())]);
+ let path = JsonPath {
+ elements: vec![PathElement::ArrayLocator(0)],
+ };
+
+ let result = mutate_json_by_path(&mut val, path, |target| match target {
+ Target::Array(arr, idx) => {
+ arr.remove(idx);
+ "removed"
+ }
+ _ => panic!("Expected Array target"),
+ });
+
+ assert_eq!(result, Some("removed"));
+ assert!(matches!(val, Val::Array(arr) if arr.is_empty()));
+ }
+
+ #[test]
+ fn test_mutate_json_none() {
+ let mut val = Val::Array(vec![]);
+ let path = JsonPath {
+ elements: vec![PathElement::ArrayLocator(0)],
+ };
+
+ let result: Option<()> = mutate_json_by_path(&mut val, path, |_| {
+ panic!("Should not be called");
+ });
+
+ assert_eq!(result, None);
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_root_strict() {
+ let path = OwnedValue::Text(LimboText {
+ value: Rc::new("$".to_string()),
+ subtype: TextSubtype::Text,
+ });
+
+ let result = json_path_from_owned_value(&path, true);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_some());
+
+ let result = result.unwrap();
+ match result.elements[..] {
+ [PathElement::Root()] => {}
+ _ => panic!("Expected root"),
+ }
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_root_non_strict() {
+ let path = OwnedValue::Text(LimboText {
+ value: Rc::new("$".to_string()),
+ subtype: TextSubtype::Text,
+ });
+
+ let result = json_path_from_owned_value(&path, false);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_some());
+
+ let result = result.unwrap();
+ match result.elements[..] {
+ [PathElement::Root()] => {}
+ _ => panic!("Expected root"),
+ }
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_named_strict() {
+ let path = OwnedValue::Text(LimboText {
+ value: Rc::new("field".to_string()),
+ subtype: TextSubtype::Text,
+ });
+
+ assert!(json_path_from_owned_value(&path, true).is_err());
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_named_non_strict() {
+ let path = OwnedValue::Text(LimboText {
+ value: Rc::new("field".to_string()),
+ subtype: TextSubtype::Text,
+ });
+
+ let result = json_path_from_owned_value(&path, false);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_some());
+
+ let result = result.unwrap();
+ match &result.elements[..] {
+ [PathElement::Root(), PathElement::Key(field)] if *field == "field" => {}
+ _ => panic!("Expected root and field"),
+ }
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_integer_strict() {
+ let path = OwnedValue::Integer(3);
+ assert!(json_path_from_owned_value(&path, true).is_err());
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_integer_non_strict() {
+ let path = OwnedValue::Integer(3);
+
+ let result = json_path_from_owned_value(&path, false);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_some());
+
+ let result = result.unwrap();
+ match &result.elements[..] {
+ [PathElement::Root(), PathElement::ArrayLocator(index)] if *index == 3 => {}
+ _ => panic!("Expected root and array locator"),
+ }
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_null_strict() {
+ let path = OwnedValue::Null;
+
+ let result = json_path_from_owned_value(&path, true);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_none());
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_null_non_strict() {
+ let path = OwnedValue::Null;
+
+ let result = json_path_from_owned_value(&path, false);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_none());
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_float_strict() {
+ let path = OwnedValue::Float(1.23);
+
+ assert!(json_path_from_owned_value(&path, true).is_err());
+ }
+
+ #[test]
+ fn test_json_path_from_owned_value_float_non_strict() {
+ let path = OwnedValue::Float(1.23);
+
+ let result = json_path_from_owned_value(&path, false);
+ assert!(result.is_ok());
+
+ let result = result.unwrap();
+ assert!(result.is_some());
+
+ let result = result.unwrap();
+ match &result.elements[..] {
+ [PathElement::Root(), PathElement::Key(field)] if *field == "1.23" => {}
+ _ => panic!("Expected root and field"),
+ }
+ }
+
+ #[test]
+ fn test_json_set_field_empty_object() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.field".to_string())),
+ OwnedValue::build_text(Rc::new("value".to_string())),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"{"field":"value"}"#.to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_replace_field() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new(r#"{"field":"old_value"}"#.to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.field".to_string())),
+ OwnedValue::build_text(Rc::new("new_value".to_string())),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"{"field":"new_value"}"#.to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_set_deeply_nested_key() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.object.doesnt.exist".to_string())),
+ OwnedValue::build_text(Rc::new("value".to_string())),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(
+ r#"{"object":{"doesnt":{"exist":"value"}}}"#.to_string()
+ ))
+ );
+ }
+
+ #[test]
+ fn test_json_set_add_value_to_empty_array() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("[]".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$[0]".to_string())),
+ OwnedValue::build_text(Rc::new("value".to_string())),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"["value"]"#.to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_add_value_to_nonexistent_array() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.some_array[0]".to_string())),
+ OwnedValue::Integer(123),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"{"some_array":[123]}"#.to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_add_value_to_array() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("[123]".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$[1]".to_string())),
+ OwnedValue::Integer(456),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new("[123,456]".to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_add_value_to_array_out_of_bounds() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("[123]".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$[200]".to_string())),
+ OwnedValue::Integer(456),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new("[123]".to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_replace_value_in_array() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("[123]".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$[0]".to_string())),
+ OwnedValue::Integer(456),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new("[456]".to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_null_path() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[OwnedValue::Null, OwnedValue::Integer(456)],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new("{}".to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_multiple_keys() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("[123]".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$[0]".to_string())),
+ OwnedValue::Integer(456),
+ OwnedValue::build_text(Rc::new("$[1]".to_string())),
+ OwnedValue::Integer(789),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new("[456,789]".to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_missing_value() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("[123]".to_string())),
+ &[OwnedValue::build_text(Rc::new("$[0]".to_string()))],
+ );
+
+ assert!(result.is_err());
+ }
+
+ #[test]
+ fn test_json_set_add_array_in_nested_object() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.object[0].field".to_string())),
+ OwnedValue::Integer(123),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"{"object":[{"field":123}]}"#.to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_add_array_in_array_in_nested_object() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.object[0][0]".to_string())),
+ OwnedValue::Integer(123),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"{"object":[[123]]}"#.to_string()))
+ );
+ }
+
+ #[test]
+ fn test_json_set_add_array_in_array_in_nested_object_out_of_bounds() {
+ let result = json_set(
+ &OwnedValue::build_text(Rc::new("{}".to_string())),
+ &[
+ OwnedValue::build_text(Rc::new("$.object[123].another".to_string())),
+ OwnedValue::build_text(Rc::new("value".to_string())),
+ OwnedValue::build_text(Rc::new("$.field".to_string())),
+ OwnedValue::build_text(Rc::new("value".to_string())),
+ ],
+ );
+
+ assert!(result.is_ok());
+
+ assert_eq!(
+ result.unwrap(),
+ OwnedValue::build_text(Rc::new(r#"{"field":"value"}"#.to_string()))
+ );
}
}
diff --git a/core/json/ser.rs b/core/json/ser.rs
index 3d5646584..680296c55 100644
--- a/core/json/ser.rs
+++ b/core/json/ser.rs
@@ -1,116 +1,157 @@
use serde::ser::{self, Serialize};
-use std::{f32, f64, num::FpCategory};
+use std::{f32, f64, io, num::FpCategory};
use crate::json::error::{Error, Result};
+#[derive(Eq, PartialEq)]
+pub enum State {
+ Empty,
+ First,
+ Rest,
+}
+
+struct Map<'a, W: 'a, F: 'a> {
+ ser: &'a mut Serializer,
+ state: State,
+}
+
/// Attempts to serialize the input as a JSON5 string (actually a JSON string).
pub fn to_string(value: &T) -> Result
where
T: Serialize,
{
- let mut serializer = Serializer {
- output: String::new(),
- };
- value.serialize(&mut serializer)?;
- Ok(serializer.output)
+ let vec = to_vec(value)?;
+ let string = String::from_utf8(vec).map_err(|err| Error::from(err.utf8_error()))?;
+ Ok(string)
}
-struct Serializer {
- output: String,
- // TODO settings for formatting (single vs double quotes, whitespace etc)
+/// Attempts to serialize the input as a JSON5 string (actually a JSON string).
+pub fn to_string_pretty(value: &T, indent: &str) -> Result
+where
+ T: Serialize,
+{
+ let vec = to_vec_pretty(value, indent)?;
+ let string = String::from_utf8(vec).map_err(|err| Error::from(err.utf8_error()))?;
+ Ok(string)
}
-impl Serializer {
- fn call_to_string(&mut self, v: &T) -> Result<()>
- where
- T: ToString,
- {
- self.output += &v.to_string();
- Ok(())
+struct Serializer {
+ writer: W,
+ formatter: F,
+}
+
+impl Serializer
+where
+ W: io::Write,
+{
+ pub fn new(writer: W) -> Self {
+ Serializer::with_formatter(writer, CompactFormatter)
}
}
-impl ser::Serializer for &mut Serializer {
+impl<'a, W> Serializer>
+where
+ W: io::Write,
+{
+ /// Creates a new JSON pretty print serializer.
+ #[inline]
+ pub fn pretty(writer: W, indent: &'a str) -> Self {
+ Serializer::with_formatter(writer, PrettyFormatter::with_indent(indent.as_bytes()))
+ }
+}
+
+impl Serializer
+where
+ W: io::Write,
+ F: Formatter,
+{
+ /// Creates a new JSON visitor whose output will be written to the writer
+ /// specified.
+ pub fn with_formatter(writer: W, formatter: F) -> Self {
+ Serializer { writer, formatter }
+ }
+}
+
+impl<'a, W, F> ser::Serializer for &'a mut Serializer
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
- type SerializeSeq = Self;
- type SerializeTuple = Self;
- type SerializeTupleStruct = Self;
- type SerializeTupleVariant = Self;
- type SerializeMap = Self;
- type SerializeStruct = Self;
- type SerializeStructVariant = Self;
+ type SerializeSeq = Map<'a, W, F>;
+ type SerializeTuple = Map<'a, W, F>;
+ type SerializeTupleStruct = Map<'a, W, F>;
+ type SerializeTupleVariant = Map<'a, W, F>;
+ type SerializeMap = Map<'a, W, F>;
+ type SerializeStruct = Map<'a, W, F>;
+ type SerializeStructVariant = Map<'a, W, F>;
fn serialize_bool(self, v: bool) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_bool(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_i8(self, v: i8) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_i8(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_i16(self, v: i16) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_i16(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_i32(self, v: i32) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_i32(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_i64(self, v: i64) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_i64(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_u8(self, v: u8) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_u8(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_u16(self, v: u16) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_u16(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_u32(self, v: u32) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_u32(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_u64(self, v: u64) -> Result<()> {
- self.call_to_string(&v)
+ self.formatter
+ .write_u64(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_f32(self, v: f32) -> Result<()> {
- match v.classify() {
- FpCategory::Nan => self.output += "null",
- FpCategory::Infinite => {
- let infinity = if v.is_sign_negative() {
- "-9e999"
- } else {
- "9e999"
- };
- self.output += infinity
- }
- _ => self.output += &v.to_string(),
- }
- Ok(())
+ self.formatter
+ .write_f32(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_f64(self, v: f64) -> Result<()> {
- match v.classify() {
- FpCategory::Nan => self.output += "null",
- FpCategory::Infinite => {
- let infinity = if v.is_sign_negative() {
- "-9e999"
- } else {
- "9e999"
- };
- self.output += infinity
- }
- _ => {
- let str = &format!("{:.1}", v);
- self.output += str
- }
- }
- Ok(())
+ self.formatter
+ .write_f64(&mut self.writer, v)
+ .map_err(Error::from)
}
fn serialize_char(self, v: char) -> Result<()> {
@@ -120,10 +161,7 @@ impl ser::Serializer for &mut Serializer {
}
fn serialize_str(self, v: &str) -> Result<()> {
- self.output += "\"";
- self.output += &escape(v);
- self.output += "\"";
- Ok(())
+ format_escaped_str(&mut self.writer, &mut self.formatter, v).map_err(Error::from)
}
fn serialize_bytes(self, _v: &[u8]) -> Result<()> {
@@ -142,8 +180,9 @@ impl ser::Serializer for &mut Serializer {
}
fn serialize_unit(self) -> Result<()> {
- self.output += "null";
- Ok(())
+ self.formatter
+ .write_null(&mut self.writer)
+ .map_err(Error::from)
}
fn serialize_unit_struct(self, _name: &'static str) -> Result<()> {
@@ -176,17 +215,47 @@ impl ser::Serializer for &mut Serializer {
where
T: ?Sized + Serialize,
{
- self.output += "{";
- variant.serialize(&mut *self)?; // TODO drop the quotes where possible
- self.output += ":";
+ self.formatter
+ .begin_object(&mut self.writer)
+ .map_err(Error::from)?;
+ self.formatter
+ .begin_object_key(&mut self.writer, true)
+ .map_err(Error::from)?;
+ self.serialize_str(variant)?;
+ self.formatter
+ .end_object_key(&mut self.writer)
+ .map_err(Error::from)?;
+ self.formatter
+ .begin_object_value(&mut self.writer)
+ .map_err(Error::from)?;
value.serialize(&mut *self)?;
- self.output += "}";
- Ok(())
+ self.formatter
+ .end_object_value(&mut self.writer)
+ .map_err(Error::from)?;
+ self.formatter
+ .end_object(&mut self.writer)
+ .map_err(Error::from)
}
- fn serialize_seq(self, _len: Option) -> Result {
- self.output += "[";
- Ok(self)
+ fn serialize_seq(self, len: Option) -> Result {
+ self.formatter
+ .begin_array(&mut self.writer)
+ .map_err(Error::from)?;
+
+ if len == Some(0) {
+ self.formatter
+ .end_array(&mut self.writer)
+ .map_err(Error::from)?;
+ Ok(Map {
+ ser: self,
+ state: State::Empty,
+ })
+ } else {
+ Ok(Map {
+ ser: self,
+ state: State::First,
+ })
+ }
}
fn serialize_tuple(self, len: usize) -> Result {
@@ -206,17 +275,34 @@ impl ser::Serializer for &mut Serializer {
_name: &'static str,
_variant_index: u32,
variant: &'static str,
- _len: usize,
+ len: usize,
) -> Result {
- self.output += "{";
- variant.serialize(&mut *self)?;
- self.output += ":[";
- Ok(self)
+ self.formatter.begin_object(&mut self.writer)?;
+ self.formatter.begin_object_key(&mut self.writer, true)?;
+ self.serialize_str(variant)?;
+ self.formatter.end_object_key(&mut self.writer)?;
+ self.formatter.begin_object_value(&mut self.writer)?;
+ self.serialize_seq(Some(len))
}
- fn serialize_map(self, _len: Option) -> Result {
- self.output += "{";
- Ok(self)
+ fn serialize_map(self, len: Option) -> Result {
+ self.formatter
+ .begin_object(&mut self.writer)
+ .map_err(Error::from)?;
+ if len == Some(0) {
+ self.formatter
+ .end_object(&mut self.writer)
+ .map_err(Error::from)?;
+ Ok(Map {
+ ser: self,
+ state: State::Empty,
+ })
+ } else {
+ Ok(Map {
+ ser: self,
+ state: State::First,
+ })
+ }
}
fn serialize_struct(self, _name: &'static str, len: usize) -> Result {
@@ -228,16 +314,30 @@ impl ser::Serializer for &mut Serializer {
_name: &'static str,
_variant_index: u32,
variant: &'static str,
- _len: usize,
+ len: usize,
) -> Result {
- self.output += "{";
- variant.serialize(&mut *self)?;
- self.output += ":{";
- Ok(self)
+ self.formatter
+ .begin_object(&mut self.writer)
+ .map_err(Error::from)?;
+ self.formatter
+ .begin_object_key(&mut self.writer, true)
+ .map_err(Error::from)?;
+ self.serialize_str(variant).map_err(Error::from)?;
+ self.formatter
+ .end_object_key(&mut self.writer)
+ .map_err(Error::from)?;
+ self.formatter
+ .begin_object_value(&mut self.writer)
+ .map_err(Error::from)?;
+ self.serialize_map(Some(len))
}
}
-impl ser::SerializeSeq for &mut Serializer {
+impl ser::SerializeSeq for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -245,19 +345,36 @@ impl ser::SerializeSeq for &mut Serializer {
where
T: ?Sized + Serialize,
{
- if !self.output.ends_with('[') {
- self.output += ",";
+ self.ser
+ .formatter
+ .begin_array_value(&mut self.ser.writer, self.state == State::First)
+ .map_err(Error::from)?;
+
+ self.state = State::Rest;
+ value.serialize(&mut *self.ser).map_err(Error::from)?;
+ self.ser
+ .formatter
+ .end_array_value(&mut self.ser.writer)
+ .map_err(Error::from)
+ }
+
+ fn end(self) -> Result<()> {
+ match self.state {
+ State::Empty => Ok(()),
+ _ => self
+ .ser
+ .formatter
+ .end_array(&mut self.ser.writer)
+ .map_err(Error::from),
}
- value.serialize(&mut **self)
- }
-
- fn end(self) -> Result<()> {
- self.output += "]";
- Ok(())
}
}
-impl ser::SerializeTuple for &mut Serializer {
+impl ser::SerializeTuple for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -273,7 +390,11 @@ impl ser::SerializeTuple for &mut Serializer {
}
}
-impl ser::SerializeTupleStruct for &mut Serializer {
+impl ser::SerializeTupleStruct for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -289,7 +410,11 @@ impl ser::SerializeTupleStruct for &mut Serializer {
}
}
-impl ser::SerializeTupleVariant for &mut Serializer {
+impl ser::SerializeTupleVariant for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -301,12 +426,30 @@ impl ser::SerializeTupleVariant for &mut Serializer {
}
fn end(self) -> Result<()> {
- self.output += "]}";
- Ok(())
+ match self.state {
+ State::Empty => {}
+ _ => self
+ .ser
+ .formatter
+ .end_array(&mut self.ser.writer)
+ .map_err(Error::from)?,
+ };
+ self.ser
+ .formatter
+ .end_object_value(&mut self.ser.writer)
+ .map_err(Error::from)?;
+ self.ser
+ .formatter
+ .end_object(&mut self.ser.writer)
+ .map_err(Error::from)
}
}
-impl ser::SerializeMap for &mut Serializer {
+impl ser::SerializeMap for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -314,27 +457,54 @@ impl ser::SerializeMap for &mut Serializer {
where
T: ?Sized + Serialize,
{
- if !self.output.ends_with('{') {
- self.output += ",";
- }
- key.serialize(&mut **self)
+ self.ser
+ .formatter
+ .begin_object_key(&mut self.ser.writer, self.state == State::First)
+ .map_err(Error::from)?;
+ self.state = State::Rest;
+
+ key.serialize(&mut *self.ser)?;
+
+ self.ser
+ .formatter
+ .end_object_key(&mut self.ser.writer)
+ .map_err(Error::from)
}
fn serialize_value(&mut self, value: &T) -> Result<()>
where
T: ?Sized + Serialize,
{
- self.output += ":";
- value.serialize(&mut **self)
+ self.ser
+ .formatter
+ .begin_object_value(&mut self.ser.writer)
+ .map_err(Error::from)?;
+
+ value.serialize(&mut *self.ser)?;
+
+ self.ser
+ .formatter
+ .end_object_value(&mut self.ser.writer)
+ .map_err(Error::from)
}
fn end(self) -> Result<()> {
- self.output += "}";
- Ok(())
+ match self.state {
+ State::Empty => Ok(()),
+ _ => self
+ .ser
+ .formatter
+ .end_object(&mut self.ser.writer)
+ .map_err(Error::from),
+ }
}
}
-impl ser::SerializeStruct for &mut Serializer {
+impl ser::SerializeStruct for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -351,7 +521,11 @@ impl ser::SerializeStruct for &mut Serializer {
}
}
-impl ser::SerializeStructVariant for &mut Serializer {
+impl ser::SerializeStructVariant for Map<'_, W, F>
+where
+ W: io::Write,
+ F: Formatter,
+{
type Ok = ();
type Error = Error;
@@ -363,22 +537,667 @@ impl ser::SerializeStructVariant for &mut Serializer {
}
fn end(self) -> Result<()> {
- self.output += "}}";
+ match self.state {
+ State::Empty => {}
+ _ => self
+ .ser
+ .formatter
+ .end_object(&mut self.ser.writer)
+ .map_err(Error::from)?,
+ };
+ self.ser
+ .formatter
+ .end_object_value(&mut self.ser.writer)
+ .map_err(Error::from)?;
+ self.ser
+ .formatter
+ .end_object(&mut self.ser.writer)
+ .map_err(Error::from)
+ }
+}
+
+pub fn to_writer(writer: W, value: &T) -> Result<()>
+where
+ W: io::Write,
+ T: ?Sized + Serialize,
+{
+ let mut ser = Serializer::new(writer);
+ value.serialize(&mut ser)
+}
+
+pub fn to_vec(value: &T) -> Result>
+where
+ T: ?Sized + Serialize,
+{
+ let mut writer = Vec::with_capacity(128);
+ to_writer(&mut writer, value)?;
+ Ok(writer)
+}
+
+pub fn to_writer_pretty(writer: W, value: &T, indent: &str) -> Result<()>
+where
+ W: io::Write,
+ T: ?Sized + Serialize,
+{
+ let mut ser = Serializer::pretty(writer, indent);
+ value.serialize(&mut ser)
+}
+
+pub fn to_vec_pretty(value: &T, indent: &str) -> Result>
+where
+ T: ?Sized + Serialize,
+{
+ let mut writer = Vec::with_capacity(128);
+ to_writer_pretty(&mut writer, value, indent)?;
+ Ok(writer)
+}
+
+/// Represents a character escape code in a type-safe manner.
+pub enum CharEscape {
+ /// An escaped quote `"`
+ Quote,
+ /// An escaped reverse solidus `\`
+ ReverseSolidus,
+ /// An escaped solidus `/`
+ Solidus,
+ /// An escaped backspace character (usually escaped as `\b`)
+ Backspace,
+ /// An escaped form feed character (usually escaped as `\f`)
+ FormFeed,
+ /// An escaped line feed character (usually escaped as `\n`)
+ LineFeed,
+ /// An escaped carriage return character (usually escaped as `\r`)
+ CarriageReturn,
+ /// An escaped tab character (usually escaped as `\t`)
+ Tab,
+ /// An escaped ASCII plane control character (usually escaped as
+ /// `\u00XX` where `XX` are two hex characters)
+ AsciiControl(u8),
+}
+
+impl CharEscape {
+ fn from_escape_table(escape: u8, byte: u8) -> CharEscape {
+ match escape {
+ self::BB => CharEscape::Backspace,
+ self::TT => CharEscape::Tab,
+ self::NN => CharEscape::LineFeed,
+ self::FF => CharEscape::FormFeed,
+ self::RR => CharEscape::CarriageReturn,
+ self::QU => CharEscape::Quote,
+ self::BS => CharEscape::ReverseSolidus,
+ self::UU => CharEscape::AsciiControl(byte),
+ _ => unreachable!(),
+ }
+ }
+}
+
+const BB: u8 = b'b'; // \x08
+const TT: u8 = b't'; // \x09
+const NN: u8 = b'n'; // \x0A
+const FF: u8 = b'f'; // \x0C
+const RR: u8 = b'r'; // \x0D
+const QU: u8 = b'"'; // \x22
+const BS: u8 = b'\\'; // \x5C
+const UU: u8 = b'u'; // \x00...\x1F except the ones above
+const __: u8 = 0;
+
+pub trait Formatter {
+ /// Writes a `null` value to the specified writer.
+ fn write_null(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"null")
+ }
+
+ /// Writes a `true` or `false` value to the specified writer.
+ fn write_bool(&mut self, writer: &mut W, value: bool) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ let s = if value {
+ b"true" as &[u8]
+ } else {
+ b"false" as &[u8]
+ };
+ writer.write_all(s)
+ }
+
+ /// Writes an integer value like `-123` to the specified writer.
+ fn write_i8(&mut self, writer: &mut W, value: i8) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `-123` to the specified writer.
+ fn write_i16(&mut self, writer: &mut W, value: i16) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `-123` to the specified writer.
+ fn write_i32(&mut self, writer: &mut W, value: i32) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `-123` to the specified writer.
+ fn write_i64(&mut self, writer: &mut W, value: i64) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `-123` to the specified writer.
+ fn write_i128(&mut self, writer: &mut W, value: i128) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `123` to the specified writer.
+ fn write_u8(&mut self, writer: &mut W, value: u8) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `123` to the specified writer.
+ fn write_u16(&mut self, writer: &mut W, value: u16) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `123` to the specified writer.
+ fn write_u32(&mut self, writer: &mut W, value: u32) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `123` to the specified writer.
+ fn write_u64(&mut self, writer: &mut W, value: u64) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes an integer value like `123` to the specified writer.
+ fn write_u128(&mut self, writer: &mut W, value: u128) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.to_string().as_bytes())
+ }
+
+ /// Writes a floating point value like `-31.26e+12` to the specified writer.
+ fn write_f32(&mut self, writer: &mut W, value: f32) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ match value.classify() {
+ FpCategory::Nan => {
+ self.write_null(writer)?;
+ }
+ FpCategory::Infinite => {
+ let infinity = if value.is_sign_negative() {
+ "-9e999"
+ } else {
+ "9e999"
+ };
+ writer.write_all(infinity.as_bytes())?;
+ }
+ _ => {
+ writer.write_all(value.to_string().as_bytes())?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Writes a floating point value like `-31.26e+12` to the specified writer.
+ fn write_f64(&mut self, writer: &mut W, value: f64) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ match value.classify() {
+ FpCategory::Nan => {
+ self.write_null(writer)?;
+ }
+ FpCategory::Infinite => {
+ let infinity = if value.is_sign_negative() {
+ "-9e999"
+ } else {
+ "9e999"
+ };
+ writer.write_all(infinity.as_bytes())?;
+ }
+ _ => {
+ // let mut buffer = ryu::Buffer::new();
+ // let s = buffer.format_finite(value);
+
+ // This the previous implementation present in the package
+ // However, serde_json does it differently above.
+ // Not sure if there if its done like this because of the precision
+ let s = &format!("{:.1}", value);
+ writer.write_all(s.as_bytes())?;
+ }
+ }
+ Ok(())
+ }
+
+ /// Writes a number that has already been rendered to a string.
+ fn write_number_str(&mut self, writer: &mut W, value: &str) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(value.as_bytes())
+ }
+
+ /// Called before each series of `write_string_fragment` and
+ /// `write_char_escape`. Writes a `"` to the specified writer.
+ fn begin_string(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"\"")
+ }
+
+ /// Called after each series of `write_string_fragment` and
+ /// `write_char_escape`. Writes a `"` to the specified writer.
+ fn end_string(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"\"")
+ }
+
+ /// Writes a string fragment that doesn't need any escaping to the
+ /// specified writer.
+ fn write_string_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(fragment.as_bytes())
+ }
+
+ /// Writes a character escape code to the specified writer.
+ fn write_char_escape(&mut self, writer: &mut W, char_escape: CharEscape) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ use self::CharEscape::*;
+
+ let s = match char_escape {
+ Quote => b"\\\"",
+ ReverseSolidus => b"\\\\",
+ Solidus => b"\\/",
+ Backspace => b"\\b",
+ FormFeed => b"\\f",
+ LineFeed => b"\\n",
+ CarriageReturn => b"\\r",
+ Tab => b"\\t",
+ AsciiControl(byte) => {
+ static HEX_DIGITS: [u8; 16] = *b"0123456789abcdef";
+ let bytes = &[
+ b'\\',
+ b'u',
+ b'0',
+ b'0',
+ HEX_DIGITS[(byte >> 4) as usize],
+ HEX_DIGITS[(byte & 0xF) as usize],
+ ];
+ return writer.write_all(bytes);
+ }
+ };
+
+ writer.write_all(s)
+ }
+
+ /// Writes the representation of a byte array. Formatters can choose whether
+ /// to represent bytes as a JSON array of integers (the default), or some
+ /// JSON string encoding like hex or base64.
+ fn write_byte_array(&mut self, writer: &mut W, value: &[u8]) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.begin_array(writer)?;
+ let mut first = true;
+ for byte in value {
+ self.begin_array_value(writer, first)?;
+ self.write_u8(writer, *byte)?;
+ self.end_array_value(writer)?;
+ first = false;
+ }
+ self.end_array(writer)
+ }
+
+ /// Called before every array. Writes a `[` to the specified
+ /// writer.
+ fn begin_array(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"[")
+ }
+
+ /// Called after every array. Writes a `]` to the specified
+ /// writer.
+ fn end_array(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"]")
+ }
+
+ /// Called before every array value. Writes a `,` if needed to
+ /// the specified writer.
+ fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ if first {
+ Ok(())
+ } else {
+ writer.write_all(b",")
+ }
+ }
+
+ /// Called after every array value.
+ fn end_array_value(&mut self, _writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ Ok(())
+ }
+
+ /// Called before every object. Writes a `{` to the specified
+ /// writer.
+ fn begin_object(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"{")
+ }
+
+ /// Called after every object. Writes a `}` to the specified
+ /// writer.
+ fn end_object(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b"}")
+ }
+
+ /// Called before every object key.
+ fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ if first {
+ Ok(())
+ } else {
+ writer.write_all(b",")
+ }
+ }
+
+ /// Called after every object key. A `:` should be written to the
+ /// specified writer by either this method or
+ /// `begin_object_value`.
+ fn end_object_key(&mut self, _writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ Ok(())
+ }
+
+ /// Called before every object value. A `:` should be written to
+ /// the specified writer by either this method or
+ /// `end_object_key`.
+ fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b":")
+ }
+
+ /// Called after every object value.
+ fn end_object_value(&mut self, _writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ Ok(())
+ }
+
+ /// Writes a raw JSON fragment that doesn't need any escaping to the
+ /// specified writer.
+ fn write_raw_fragment(&mut self, writer: &mut W, fragment: &str) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(fragment.as_bytes())
+ }
+}
+
+fn format_escaped_str(writer: &mut W, formatter: &mut F, value: &str) -> io::Result<()>
+where
+ W: ?Sized + io::Write,
+ F: ?Sized + Formatter,
+{
+ formatter.begin_string(writer)?;
+ format_escaped_str_contents(writer, formatter, value)?;
+ formatter.end_string(writer)
+}
+
+fn format_escaped_str_contents(
+ writer: &mut W,
+ formatter: &mut F,
+ value: &str,
+) -> io::Result<()>
+where
+ W: ?Sized + io::Write,
+ F: ?Sized + Formatter,
+{
+ let bytes = value.as_bytes();
+
+ let mut start = 0;
+
+ for (i, &byte) in bytes.iter().enumerate() {
+ let escape = self::ESCAPE[byte as usize];
+ if escape == 0 {
+ continue;
+ }
+
+ if start < i {
+ formatter.write_string_fragment(writer, &value[start..i])?;
+ }
+
+ let char_escape = CharEscape::from_escape_table(escape, byte);
+ formatter.write_char_escape(writer, char_escape)?;
+
+ start = i + 1;
+ }
+
+ if start == bytes.len() {
+ return Ok(());
+ }
+
+ formatter.write_string_fragment(writer, &value[start..])
+}
+
+// Lookup table of escape sequences. A value of b'x' at index i means that byte
+// i is escaped as "\x" in JSON. A value of 0 means that byte i is not escaped.
+static ESCAPE: [u8; 256] = [
+ // 1 2 3 4 5 6 7 8 9 A B C D E F
+ UU, UU, UU, UU, UU, UU, UU, UU, BB, TT, NN, UU, FF, RR, UU, UU, // 0
+ UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, UU, // 1
+ __, __, QU, __, __, __, __, __, __, __, __, __, __, __, __, __, // 2
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 3
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 4
+ __, __, __, __, __, __, __, __, __, __, __, __, BS, __, __, __, // 5
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 6
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 7
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 8
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // 9
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // A
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // B
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // C
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // D
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // E
+ __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, __, // F
+];
+
+/// This structure compacts a JSON value with no extra whitespace.
+#[derive(Clone, Debug)]
+pub struct CompactFormatter;
+
+impl Formatter for CompactFormatter {}
+
+/// This structure pretty prints a JSON value to make it human readable.
+#[derive(Clone, Debug)]
+pub struct PrettyFormatter<'a> {
+ current_indent: usize,
+ has_value: bool,
+ indent: &'a [u8],
+}
+
+impl<'a> PrettyFormatter<'a> {
+ /// Construct a pretty printer formatter that defaults to using two spaces for indentation.
+ pub fn new() -> Self {
+ PrettyFormatter::with_indent(b" ")
+ }
+
+ /// Construct a pretty printer formatter that uses the `indent` string for indentation.
+ pub fn with_indent(indent: &'a [u8]) -> Self {
+ PrettyFormatter {
+ current_indent: 0,
+ has_value: false,
+ indent,
+ }
+ }
+}
+
+impl Default for PrettyFormatter<'_> {
+ fn default() -> Self {
+ PrettyFormatter::new()
+ }
+}
+
+impl Formatter for PrettyFormatter<'_> {
+ #[inline]
+ fn begin_array(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.current_indent += 1;
+ self.has_value = false;
+ writer.write_all(b"[")
+ }
+
+ #[inline]
+ fn end_array(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.current_indent -= 1;
+
+ if self.has_value {
+ writer.write_all(b"\n")?;
+ indent(writer, self.current_indent, self.indent)?;
+ }
+
+ writer.write_all(b"]")
+ }
+
+ #[inline]
+ fn begin_array_value(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(if first { b"\n" } else { b",\n" })?;
+ indent(writer, self.current_indent, self.indent)
+ }
+
+ #[inline]
+ fn end_array_value(&mut self, _writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.has_value = true;
+ Ok(())
+ }
+
+ #[inline]
+ fn begin_object(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.current_indent += 1;
+ self.has_value = false;
+ writer.write_all(b"{")
+ }
+
+ #[inline]
+ fn end_object(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.current_indent -= 1;
+
+ if self.has_value {
+ writer.write_all(b"\n")?;
+ indent(writer, self.current_indent, self.indent)?;
+ }
+
+ writer.write_all(b"}")
+ }
+
+ #[inline]
+ fn begin_object_key(&mut self, writer: &mut W, first: bool) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(if first { b"\n" } else { b",\n" })?;
+ indent(writer, self.current_indent, self.indent)
+ }
+
+ #[inline]
+ fn begin_object_value(&mut self, writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ writer.write_all(b": ")
+ }
+
+ #[inline]
+ fn end_object_value(&mut self, _writer: &mut W) -> io::Result<()>
+ where
+ W: ?Sized + io::Write,
+ {
+ self.has_value = true;
Ok(())
}
}
-fn escape(v: &str) -> String {
- v.chars()
- .flat_map(|c| match c {
- '"' => vec!['\\', c],
- '\n' => vec!['\\', 'n'],
- '\r' => vec!['\\', 'r'],
- '\t' => vec!['\\', 't'],
- '\\' => vec!['\\', '\\'],
- '\u{0008}' => vec!['\\', 'b'],
- '\u{000c}' => vec!['\\', 'f'],
- c => vec![c],
- })
- .collect()
+fn indent(wr: &mut W, n: usize, s: &[u8]) -> io::Result<()>
+where
+ W: ?Sized + io::Write,
+{
+ for _ in 0..n {
+ wr.write_all(s)?;
+ }
+
+ Ok(())
}
diff --git a/core/lib.rs b/core/lib.rs
index e6c812110..c2e0f22c6 100644
--- a/core/lib.rs
+++ b/core/lib.rs
@@ -1,9 +1,11 @@
mod error;
mod ext;
mod function;
+mod info;
mod io;
#[cfg(feature = "json")]
mod json;
+pub mod mvcc;
mod parameters;
mod pseudo;
mod result;
@@ -21,16 +23,16 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
use fallible_iterator::FallibleIterator;
#[cfg(not(target_family = "wasm"))]
use libloading::{Library, Symbol};
-#[cfg(not(target_family = "wasm"))]
use limbo_ext::{ExtensionApi, ExtensionEntryPoint};
use log::trace;
+use parking_lot::RwLock;
use schema::Schema;
use sqlite3_parser::ast;
use sqlite3_parser::{ast::Cmd, lexer::sql::Parser};
use std::cell::Cell;
use std::collections::HashMap;
use std::num::NonZero;
-use std::sync::{Arc, OnceLock, RwLock};
+use std::sync::{Arc, OnceLock};
use std::{cell::RefCell, rc::Rc};
use storage::btree::btree_init_page;
#[cfg(feature = "fs")]
@@ -42,11 +44,13 @@ pub use storage::wal::WalFile;
pub use storage::wal::WalFileShared;
pub use types::Value;
use util::parse_schema_rows;
+use vdbe::builder::QueryMode;
pub use error::LimboError;
use translate::select::prepare_select_plan;
pub type Result = std::result::Result;
+use crate::storage::wal::CheckpointResult;
use crate::translate::optimizer::optimize_plan;
pub use io::OpenFlags;
pub use io::PlatformIO;
@@ -61,9 +65,10 @@ pub use storage::pager::Page;
pub use storage::pager::Pager;
pub use storage::wal::CheckpointStatus;
pub use storage::wal::Wal;
+
pub static DATABASE_VERSION: OnceLock = OnceLock::new();
-#[derive(Clone)]
+#[derive(Clone, PartialEq, Eq)]
enum TransactionState {
Write,
Read,
@@ -138,6 +143,9 @@ impl Database {
_shared_wal: shared_wal.clone(),
syms,
};
+ if let Err(e) = db.register_builtins() {
+ return Err(LimboError::ExtensionError(e));
+ }
let db = Arc::new(db);
let conn = Rc::new(Connection {
db: db.clone(),
@@ -253,10 +261,10 @@ pub struct Connection {
}
impl Connection {
- pub fn prepare(self: &Rc, sql: impl Into) -> Result {
- let sql = sql.into();
+ pub fn prepare(self: &Rc, sql: impl AsRef) -> Result {
+ let sql = sql.as_ref();
trace!("Preparing: {}", sql);
- let db = self.db.clone();
+ let db = &self.db;
let syms: &SymbolTable = &db.syms.borrow();
let mut parser = Parser::new(sql.as_bytes());
let cmd = parser.next()?;
@@ -270,6 +278,7 @@ impl Connection {
self.pager.clone(),
Rc::downgrade(self),
syms,
+ QueryMode::Normal,
)?);
Ok(Statement::new(program, self.pager.clone()))
}
@@ -281,8 +290,8 @@ impl Connection {
}
}
- pub fn query(self: &Rc, sql: impl Into) -> Result> {
- let sql = sql.into();
+ pub fn query(self: &Rc, sql: impl AsRef) -> Result> {
+ let sql = sql.as_ref();
trace!("Querying: {}", sql);
let mut parser = Parser::new(sql.as_bytes());
let cmd = parser.next()?;
@@ -304,6 +313,7 @@ impl Connection {
self.pager.clone(),
Rc::downgrade(self),
syms,
+ QueryMode::Normal,
)?);
let stmt = Statement::new(program, self.pager.clone());
Ok(Some(stmt))
@@ -316,6 +326,7 @@ impl Connection {
self.pager.clone(),
Rc::downgrade(self),
syms,
+ QueryMode::Explain,
)?;
program.explain();
Ok(None)
@@ -328,7 +339,7 @@ impl Connection {
*select,
&self.db.syms.borrow(),
)?;
- optimize_plan(&mut plan)?;
+ optimize_plan(&mut plan, &self.schema.borrow())?;
println!("{}", plan);
}
_ => todo!(),
@@ -342,9 +353,9 @@ impl Connection {
QueryRunner::new(self, sql)
}
- pub fn execute(self: &Rc, sql: impl Into) -> Result<()> {
- let sql = sql.into();
- let db = self.db.clone();
+ pub fn execute(self: &Rc, sql: impl AsRef) -> Result<()> {
+ let sql = sql.as_ref();
+ let db = &self.db;
let syms: &SymbolTable = &db.syms.borrow();
let mut parser = Parser::new(sql.as_bytes());
let cmd = parser.next()?;
@@ -358,6 +369,7 @@ impl Connection {
self.pager.clone(),
Rc::downgrade(self),
syms,
+ QueryMode::Explain,
)?;
program.explain();
}
@@ -370,6 +382,7 @@ impl Connection {
self.pager.clone(),
Rc::downgrade(self),
syms,
+ QueryMode::Normal,
)?;
let mut state =
@@ -390,9 +403,9 @@ impl Connection {
Ok(())
}
- pub fn checkpoint(&self) -> Result<()> {
- self.pager.clear_page_cache();
- Ok(())
+ pub fn checkpoint(&self) -> Result {
+ let checkpoint_result = self.pager.clear_page_cache();
+ Ok(checkpoint_result)
}
#[cfg(not(target_family = "wasm"))]
@@ -405,7 +418,7 @@ impl Connection {
loop {
// TODO: make this async?
match self.pager.checkpoint()? {
- CheckpointStatus::Done => {
+ CheckpointStatus::Done(_) => {
return Ok(());
}
CheckpointStatus::IO => {
@@ -455,19 +468,7 @@ impl Statement {
}
pub fn step(&mut self) -> Result> {
- let result = self.program.step(&mut self.state, self.pager.clone())?;
- match result {
- vdbe::StepResult::Row(row) => Ok(StepResult::Row(Row { values: row.values })),
- vdbe::StepResult::IO => Ok(StepResult::IO),
- vdbe::StepResult::Done => Ok(StepResult::Done),
- vdbe::StepResult::Interrupt => Ok(StepResult::Interrupt),
- vdbe::StepResult::Busy => Ok(StepResult::Busy),
- }
- }
-
- pub fn query(&mut self) -> Result {
- let stmt = Statement::new(self.program.clone(), self.pager.clone());
- Ok(stmt)
+ self.program.step(&mut self.state, self.pager.clone())
}
pub fn columns(&self) -> &[String] {
@@ -491,19 +492,9 @@ impl Statement {
}
}
-#[derive(PartialEq)]
-pub enum StepResult<'a> {
- Row(Row<'a>),
- IO,
- Done,
- Interrupt,
- Busy,
-}
+pub type StepResult<'a> = vdbe::StepResult<'a>;
-#[derive(PartialEq)]
-pub struct Row<'a> {
- pub values: Vec>,
-}
+pub type Row<'a> = types::Record<'a>;
impl<'a> Row<'a> {
pub fn get + 'a>(&self, idx: usize) -> Result {
@@ -557,7 +548,6 @@ impl SymbolTable {
pub fn new() -> Self {
Self {
functions: HashMap::new(),
- // TODO: wasm libs will be very different
#[cfg(not(target_family = "wasm"))]
extensions: Vec::new(),
}
diff --git a/core/mvcc/clock.rs b/core/mvcc/clock.rs
new file mode 100644
index 000000000..7bab1fe5d
--- /dev/null
+++ b/core/mvcc/clock.rs
@@ -0,0 +1,31 @@
+use std::sync::atomic::{AtomicU64, Ordering};
+
+/// Logical clock.
+pub trait LogicalClock {
+ fn get_timestamp(&self) -> u64;
+ fn reset(&self, ts: u64);
+}
+
+/// A node-local clock backed by an atomic counter.
+#[derive(Debug, Default)]
+pub struct LocalClock {
+ ts_sequence: AtomicU64,
+}
+
+impl LocalClock {
+ pub fn new() -> Self {
+ Self {
+ ts_sequence: AtomicU64::new(0),
+ }
+ }
+}
+
+impl LogicalClock for LocalClock {
+ fn get_timestamp(&self) -> u64 {
+ self.ts_sequence.fetch_add(1, Ordering::SeqCst)
+ }
+
+ fn reset(&self, ts: u64) {
+ self.ts_sequence.store(ts, Ordering::SeqCst);
+ }
+}
diff --git a/core/mvcc/cursor.rs b/core/mvcc/cursor.rs
new file mode 100644
index 000000000..4d120e214
--- /dev/null
+++ b/core/mvcc/cursor.rs
@@ -0,0 +1,67 @@
+use serde::de::DeserializeOwned;
+use serde::Serialize;
+
+use crate::mvcc::clock::LogicalClock;
+use crate::mvcc::database::{Database, Result, Row, RowID};
+use std::fmt::Debug;
+
+#[derive(Debug)]
+pub struct ScanCursor<
+ 'a,
+ Clock: LogicalClock,
+ T: Sync + Send + Clone + Serialize + DeserializeOwned + Debug,
+> {
+ pub db: &'a Database,
+ pub row_ids: Vec,
+ pub index: usize,
+ tx_id: u64,
+}
+
+impl<
+ 'a,
+ Clock: LogicalClock,
+ T: Sync + Send + Clone + Serialize + DeserializeOwned + Debug + 'static,
+ > ScanCursor<'a, Clock, T>
+{
+ pub fn new(
+ db: &'a Database,
+ tx_id: u64,
+ table_id: u64,
+ ) -> Result> {
+ let row_ids = db.scan_row_ids_for_table(table_id)?;
+ Ok(Self {
+ db,
+ tx_id,
+ row_ids,
+ index: 0,
+ })
+ }
+
+ pub fn current_row_id(&self) -> Option {
+ if self.index >= self.row_ids.len() {
+ return None;
+ }
+ Some(self.row_ids[self.index])
+ }
+
+ pub fn current_row(&self) -> Result>> {
+ if self.index >= self.row_ids.len() {
+ return Ok(None);
+ }
+ let id = self.row_ids[self.index];
+ self.db.read(self.tx_id, id)
+ }
+
+ pub fn close(self) -> Result<()> {
+ Ok(())
+ }
+
+ pub fn forward(&mut self) -> bool {
+ self.index += 1;
+ self.index < self.row_ids.len()
+ }
+
+ pub fn is_empty(&self) -> bool {
+ self.index >= self.row_ids.len()
+ }
+}
diff --git a/core/mvcc/database/mod.rs b/core/mvcc/database/mod.rs
new file mode 100644
index 000000000..6fa8420f3
--- /dev/null
+++ b/core/mvcc/database/mod.rs
@@ -0,0 +1,810 @@
+use crate::mvcc::clock::LogicalClock;
+use crate::mvcc::errors::DatabaseError;
+use crate::mvcc::persistent_storage::Storage;
+use crossbeam_skiplist::{SkipMap, SkipSet};
+use std::fmt::Debug;
+use std::sync::atomic::{AtomicU64, Ordering};
+use std::sync::RwLock;
+
+pub type Result = std::result::Result;
+
+#[cfg(test)]
+mod tests;
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq, PartialOrd, Ord, Hash)]
+pub struct RowID {
+ pub table_id: u64,
+ pub row_id: u64,
+}
+
+#[derive(Clone, Debug, PartialEq, PartialOrd)]
+
+pub struct Row {
+ pub id: RowID,
+ pub data: T,
+}
+
+/// A row version.
+#[derive(Clone, Debug, PartialEq)]
+pub struct RowVersion {
+ begin: TxTimestampOrID,
+ end: Option,
+ row: Row,
+}
+
+pub type TxID = u64;
+
+/// A log record contains all the versions inserted and deleted by a transaction.
+#[derive(Clone, Debug)]
+pub struct LogRecord {
+ pub(crate) tx_timestamp: TxID,
+ row_versions: Vec>,
+}
+
+impl LogRecord {
+ fn new(tx_timestamp: TxID) -> Self {
+ Self {
+ tx_timestamp,
+ row_versions: Vec::new(),
+ }
+ }
+}
+
+/// A transaction timestamp or ID.
+///
+/// Versions either track a timestamp or a transaction ID, depending on the
+/// phase of the transaction. During the active phase, new versions track the
+/// transaction ID in the `begin` and `end` fields. After a transaction commits,
+/// versions switch to tracking timestamps.
+#[derive(Clone, Debug, PartialEq, PartialOrd)]
+enum TxTimestampOrID {
+ Timestamp(u64),
+ TxID(TxID),
+}
+
+/// Transaction
+#[derive(Debug)]
+pub struct Transaction {
+ /// The state of the transaction.
+ state: AtomicTransactionState,
+ /// The transaction ID.
+ tx_id: u64,
+ /// The transaction begin timestamp.
+ begin_ts: u64,
+ /// The transaction write set.
+ write_set: SkipSet,
+ /// The transaction read set.
+ read_set: SkipSet,
+}
+
+impl Transaction {
+ fn new(tx_id: u64, begin_ts: u64) -> Transaction {
+ Transaction {
+ state: TransactionState::Active.into(),
+ tx_id,
+ begin_ts,
+ write_set: SkipSet::new(),
+ read_set: SkipSet::new(),
+ }
+ }
+
+ fn insert_to_read_set(&self, id: RowID) {
+ self.read_set.insert(id);
+ }
+
+ fn insert_to_write_set(&mut self, id: RowID) {
+ self.write_set.insert(id);
+ }
+}
+
+impl std::fmt::Display for Transaction {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+ write!(
+ f,
+ "{{ state: {}, id: {}, begin_ts: {}, write_set: {:?}, read_set: {:?}",
+ self.state.load(),
+ self.tx_id,
+ self.begin_ts,
+ // FIXME: I'm sorry, we obviously shouldn't be cloning here.
+ self.write_set
+ .iter()
+ .map(|v| *v.value())
+ .collect::>(),
+ self.read_set
+ .iter()
+ .map(|v| *v.value())
+ .collect::>()
+ )
+ }
+}
+
+/// Transaction state.
+#[derive(Debug, Clone, PartialEq)]
+enum TransactionState {
+ Active,
+ Preparing,
+ Aborted,
+ Terminated,
+ Committed(u64),
+}
+
+impl TransactionState {
+ pub fn encode(&self) -> u64 {
+ match self {
+ TransactionState::Active => 0,
+ TransactionState::Preparing => 1,
+ TransactionState::Aborted => 2,
+ TransactionState::Terminated => 3,
+ TransactionState::Committed(ts) => {
+ // We only support 2*62 - 1 timestamps, because the extra bit
+ // is used to encode the type.
+ assert!(ts & 0x8000_0000_0000_0000 == 0);
+ 0x8000_0000_0000_0000 | ts
+ }
+ }
+ }
+
+ pub fn decode(v: u64) -> Self {
+ match v {
+ 0 => TransactionState::Active,
+ 1 => TransactionState::Preparing,
+ 2 => TransactionState::Aborted,
+ 3 => TransactionState::Terminated,
+ v if v & 0x8000_0000_0000_0000 != 0 => {
+ TransactionState::Committed(v & 0x7fff_ffff_ffff_ffff)
+ }
+ _ => panic!("Invalid transaction state"),
+ }
+ }
+}
+
+// Transaction state encoded into a single 64-bit atomic.
+#[derive(Debug)]
+pub(crate) struct AtomicTransactionState {
+ pub(crate) state: AtomicU64,
+}
+
+impl From for AtomicTransactionState {
+ fn from(state: TransactionState) -> Self {
+ Self {
+ state: AtomicU64::new(state.encode()),
+ }
+ }
+}
+
+impl From for TransactionState {
+ fn from(state: AtomicTransactionState) -> Self {
+ let encoded = state.state.load(Ordering::Acquire);
+ TransactionState::decode(encoded)
+ }
+}
+
+impl std::cmp::PartialEq for AtomicTransactionState {
+ fn eq(&self, other: &TransactionState) -> bool {
+ &self.load() == other
+ }
+}
+
+impl std::fmt::Display for TransactionState {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::result::Result<(), std::fmt::Error> {
+ match self {
+ TransactionState::Active => write!(f, "Active"),
+ TransactionState::Preparing => write!(f, "Preparing"),
+ TransactionState::Committed(ts) => write!(f, "Committed({ts})"),
+ TransactionState::Aborted => write!(f, "Aborted"),
+ TransactionState::Terminated => write!(f, "Terminated"),
+ }
+ }
+}
+
+impl AtomicTransactionState {
+ fn store(&self, state: TransactionState) {
+ self.state.store(state.encode(), Ordering::Release);
+ }
+
+ fn load(&self) -> TransactionState {
+ TransactionState::decode(self.state.load(Ordering::Acquire))
+ }
+}
+
+#[derive(Debug)]
+pub struct Database {
+ rows: SkipMap>>>,
+ txs: SkipMap>,
+ tx_ids: AtomicU64,
+ clock: Clock,
+ storage: Storage,
+}
+
+impl Database {
+ /// Creates a new database.
+ pub fn new(clock: Clock, storage: Storage) -> Self {
+ Self {
+ rows: SkipMap::new(),
+ txs: SkipMap::new(),
+ tx_ids: AtomicU64::new(1), // let's reserve transaction 0 for special purposes
+ clock,
+ storage,
+ }
+ }
+
+ // Extracts the begin timestamp from a transaction
+ fn get_begin_timestamp(&self, ts_or_id: &TxTimestampOrID) -> u64 {
+ match ts_or_id {
+ TxTimestampOrID::Timestamp(ts) => *ts,
+ TxTimestampOrID::TxID(tx_id) => {
+ self.txs
+ .get(tx_id)
+ .unwrap()
+ .value()
+ .read()
+ .unwrap()
+ .begin_ts
+ }
+ }
+ }
+
+ /// Inserts a new row version into the database, while making sure that
+ /// the row version is inserted in the correct order.
+ fn insert_version(&self, id: RowID, row_version: RowVersion) {
+ let versions = self.rows.get_or_insert_with(id, || RwLock::new(Vec::new()));
+ let mut versions = versions.value().write().unwrap();
+ self.insert_version_raw(&mut versions, row_version)
+ }
+
+ /// Inserts a new row version into the internal data structure for versions,
+ /// while making sure that the row version is inserted in the correct order.
+ fn insert_version_raw(&self, versions: &mut Vec>, row_version: RowVersion) {
+ // NOTICE: this is an insert a'la insertion sort, with pessimistic linear complexity.
+ // However, we expect the number of versions to be nearly sorted, so we deem it worthy
+ // to search linearly for the insertion point instead of paying the price of using
+ // another data structure, e.g. a BTreeSet. If it proves to be too quadratic empirically,
+ // we can either switch to a tree-like structure, or at least use partition_point()
+ // which performs a binary search for the insertion point.
+ let position = versions
+ .iter()
+ .rposition(|v| {
+ self.get_begin_timestamp(&v.begin) < self.get_begin_timestamp(&row_version.begin)
+ })
+ .map(|p| p + 1)
+ .unwrap_or(0);
+ if versions.len() - position > 3 {
+ tracing::debug!(
+ "Inserting a row version {} positions from the end",
+ versions.len() - position
+ );
+ }
+ versions.insert(position, row_version);
+ }
+
+ /// Inserts a new row into the database.
+ ///
+ /// This function inserts a new `row` into the database within the context
+ /// of the transaction `tx_id`.
+ ///
+ /// # Arguments
+ ///
+ /// * `tx_id` - the ID of the transaction in which to insert the new row.
+ /// * `row` - the row object containing the values to be inserted.
+ ///
+ pub fn insert(&self, tx_id: TxID, row: Row) -> Result<()> {
+ let tx = self
+ .txs
+ .get(&tx_id)
+ .ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
+ let mut tx = tx.value().write().unwrap();
+ assert_eq!(tx.state, TransactionState::Active);
+ let id = row.id;
+ let row_version = RowVersion {
+ begin: TxTimestampOrID::TxID(tx.tx_id),
+ end: None,
+ row,
+ };
+ tx.insert_to_write_set(id);
+ drop(tx);
+ self.insert_version(id, row_version);
+ Ok(())
+ }
+
+ /// Updates a row in the database with new values.
+ ///
+ /// This function updates an existing row in the database within the
+ /// context of the transaction `tx_id`. The `row` argument identifies the
+ /// row to be updated as `id` and contains the new values to be inserted.
+ ///
+ /// If the row identified by the `id` does not exist, this function does
+ /// nothing and returns `false`. Otherwise, the function updates the row
+ /// with the new values and returns `true`.
+ ///
+ /// # Arguments
+ ///
+ /// * `tx_id` - the ID of the transaction in which to update the new row.
+ /// * `row` - the row object containing the values to be updated.
+ ///
+ /// # Returns
+ ///
+ /// Returns `true` if the row was successfully updated, and `false` otherwise.
+ pub fn update(&self, tx_id: TxID, row: Row) -> Result {
+ if !self.delete(tx_id, row.id)? {
+ return Ok(false);
+ }
+ self.insert(tx_id, row)?;
+ Ok(true)
+ }
+
+ /// Inserts a row in the database with new values, previously deleting
+ /// any old data if it existed. Bails on a delete error, e.g. write-write conflict.
+ pub fn upsert(&self, tx_id: TxID, row: Row) -> Result<()> {
+ self.delete(tx_id, row.id)?;
+ self.insert(tx_id, row)
+ }
+
+ /// Deletes a row from the table with the given `id`.
+ ///
+ /// This function deletes an existing row `id` in the database within the
+ /// context of the transaction `tx_id`.
+ ///
+ /// # Arguments
+ ///
+ /// * `tx_id` - the ID of the transaction in which to delete the new row.
+ /// * `id` - the ID of the row to delete.
+ ///
+ /// # Returns
+ ///
+ /// Returns `true` if the row was successfully deleted, and `false` otherwise.
+ ///
+ pub fn delete(&self, tx_id: TxID, id: RowID) -> Result {
+ let row_versions_opt = self.rows.get(&id);
+ if let Some(ref row_versions) = row_versions_opt {
+ let mut row_versions = row_versions.value().write().unwrap();
+ for rv in row_versions.iter_mut().rev() {
+ let tx = self
+ .txs
+ .get(&tx_id)
+ .ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
+ let tx = tx.value().read().unwrap();
+ assert_eq!(tx.state, TransactionState::Active);
+ if is_write_write_conflict(&self.txs, &tx, rv) {
+ drop(row_versions);
+ drop(row_versions_opt);
+ drop(tx);
+ self.rollback_tx(tx_id);
+ return Err(DatabaseError::WriteWriteConflict);
+ }
+ if is_version_visible(&self.txs, &tx, rv) {
+ rv.end = Some(TxTimestampOrID::TxID(tx.tx_id));
+ drop(row_versions);
+ drop(row_versions_opt);
+ drop(tx);
+ let tx = self
+ .txs
+ .get(&tx_id)
+ .ok_or(DatabaseError::NoSuchTransactionID(tx_id))?;
+ let mut tx = tx.value().write().unwrap();
+ tx.insert_to_write_set(id);
+ return Ok(true);
+ }
+ }
+ }
+ Ok(false)
+ }
+
+ /// Retrieves a row from the table with the given `id`.
+ ///
+ /// This operation is performed within the scope of the transaction identified
+ /// by `tx_id`.
+ ///
+ /// # Arguments
+ ///
+ /// * `tx_id` - The ID of the transaction to perform the read operation in.
+ /// * `id` - The ID of the row to retrieve.
+ ///
+ /// # Returns
+ ///
+ /// Returns `Some(row)` with the row data if the row with the given `id` exists,
+ /// and `None` otherwise.
+ pub fn read(&self, tx_id: TxID, id: RowID) -> Result>> {
+ let tx = self.txs.get(&tx_id).unwrap();
+ let tx = tx.value().read().unwrap();
+ assert_eq!(tx.state, TransactionState::Active);
+ if let Some(row_versions) = self.rows.get(&id) {
+ let row_versions = row_versions.value().read().unwrap();
+ for rv in row_versions.iter().rev() {
+ if is_version_visible(&self.txs, &tx, rv) {
+ tx.insert_to_read_set(id);
+ return Ok(Some(rv.row.clone()));
+ }
+ }
+ }
+ Ok(None)
+ }
+
+ /// Gets all row ids in the database.
+ pub fn scan_row_ids(&self) -> Result> {
+ let keys = self.rows.iter().map(|entry| *entry.key());
+ Ok(keys.collect())
+ }
+
+ /// Gets all row ids in the database for a given table.
+ pub fn scan_row_ids_for_table(&self, table_id: u64) -> Result> {
+ Ok(self
+ .rows
+ .range(
+ RowID {
+ table_id,
+ row_id: 0,
+ }..RowID {
+ table_id,
+ row_id: u64::MAX,
+ },
+ )
+ .map(|entry| *entry.key())
+ .collect())
+ }
+
+ /// Begins a new transaction in the database.
+ ///
+ /// This function starts a new transaction in the database and returns a `TxID` value
+ /// that you can use to perform operations within the transaction. All changes made within the
+ /// transaction are isolated from other transactions until you commit the transaction.
+ pub fn begin_tx(&self) -> TxID {
+ let tx_id = self.get_tx_id();
+ let begin_ts = self.get_timestamp();
+ let tx = Transaction::new(tx_id, begin_ts);
+ tracing::trace!("BEGIN {tx}");
+ self.txs.insert(tx_id, RwLock::new(tx));
+ tx_id
+ }
+
+ /// Commits a transaction with the specified transaction ID.
+ ///
+ /// This function commits the changes made within the specified transaction and finalizes the
+ /// transaction. Once a transaction has been committed, all changes made within the transaction
+ /// are visible to other transactions that access the same data.
+ ///
+ /// # Arguments
+ ///
+ /// * `tx_id` - The ID of the transaction to commit.
+ pub fn commit_tx(&self, tx_id: TxID) -> Result<()> {
+ let end_ts = self.get_timestamp();
+ // NOTICE: the first shadowed tx keeps the entry alive in the map
+ // for the duration of this whole function, which is important for correctness!
+ let tx = self.txs.get(&tx_id).ok_or(DatabaseError::TxTerminated)?;
+ let tx = tx.value().write().unwrap();
+ match tx.state.load() {
+ TransactionState::Terminated => return Err(DatabaseError::TxTerminated),
+ _ => {
+ assert_eq!(tx.state, TransactionState::Active);
+ }
+ }
+ tx.state.store(TransactionState::Preparing);
+ tracing::trace!("PREPARE {tx}");
+
+ /* TODO: The code we have here is sufficient for snapshot isolation.
+ ** In order to implement serializability, we need the following steps:
+ **
+ ** 1. Validate if all read versions are still visible by inspecting the read_set
+ ** 2. Validate if there are no phantoms by walking the scans from scan_set (which we don't even have yet)
+ ** - a phantom is a version that became visible in the middle of our transaction,
+ ** but wasn't taken into account during one of the scans from the scan_set
+ ** 3. Wait for commit dependencies, which we don't even track yet...
+ ** Excerpt from what's a commit dependency and how it's tracked in the original paper:
+ ** """
+ A transaction T1 has a commit dependency on another transaction
+ T2, if T1 is allowed to commit only if T2 commits. If T2 aborts,
+ T1 must also abort, so cascading aborts are possible. T1 acquires a
+ commit dependency either by speculatively reading or speculatively ignoring a version,
+ instead of waiting for T2 to commit.
+ We implement commit dependencies by a register-and-report
+ approach: T1 registers its dependency with T2 and T2 informs T1
+ when it has committed or aborted. Each transaction T contains a
+ counter, CommitDepCounter, that counts how many unresolved
+ commit dependencies it still has. A transaction cannot commit
+ until this counter is zero. In addition, T has a Boolean variable
+ AbortNow that other transactions can set to tell T to abort. Each
+ transaction T also has a set, CommitDepSet, that stores transaction IDs
+ of the transactions that depend on T.
+ To take a commit dependency on a transaction T2, T1 increments
+ its CommitDepCounter and adds its transaction ID to T2’s CommitDepSet.
+ When T2 has committed, it locates each transaction in
+ its CommitDepSet and decrements their CommitDepCounter. If
+ T2 aborted, it tells the dependent transactions to also abort by
+ setting their AbortNow flags. If a dependent transaction is not
+ found, this means that it has already aborted.
+ Note that a transaction with commit dependencies may not have to
+ wait at all - the dependencies may have been resolved before it is
+ ready to commit. Commit dependencies consolidate all waits into
+ a single wait and postpone the wait to just before commit.
+ Some transactions may have to wait before commit.
+ Waiting raises a concern of deadlocks.
+ However, deadlocks cannot occur because an older transaction never
+ waits on a younger transaction. In
+ a wait-for graph the direction of edges would always be from a
+ younger transaction (higher end timestamp) to an older transaction
+ (lower end timestamp) so cycles are impossible.
+ """
+ ** If you're wondering when a speculative read happens, here you go:
+ ** Case 1: speculative read of TB:
+ """
+ If transaction TB is in the Preparing state, it has acquired an end
+ timestamp TS which will be V’s begin timestamp if TB commits.
+ A safe approach in this situation would be to have transaction T
+ wait until transaction TB commits. However, we want to avoid all
+ blocking during normal processing so instead we continue with
+ the visibility test and, if the test returns true, allow T to
+ speculatively read V. Transaction T acquires a commit dependency on
+ TB, restricting the serialization order of the two transactions. That
+ is, T is allowed to commit only if TB commits.
+ """
+ ** Case 2: speculative ignore of TE:
+ """
+ If TE’s state is Preparing, it has an end timestamp TS that will become
+ the end timestamp of V if TE does commit. If TS is greater than the read
+ time RT, it is obvious that V will be visible if TE commits. If TE
+ aborts, V will still be visible, because any transaction that updates
+ V after TE has aborted will obtain an end timestamp greater than
+ TS. If TS is less than RT, we have a more complicated situation:
+ if TE commits, V will not be visible to T but if TE aborts, it will
+ be visible. We could handle this by forcing T to wait until TE
+ commits or aborts but we want to avoid all blocking during normal processing.
+ Instead we allow T to speculatively ignore V and
+ proceed with its processing. Transaction T acquires a commit
+ dependency (see Section 2.7) on TE, that is, T is allowed to commit
+ only if TE commits.
+ """
+ */
+ tx.state.store(TransactionState::Committed(end_ts));
+ tracing::trace!("COMMIT {tx}");
+ let tx_begin_ts = tx.begin_ts;
+ let write_set: Vec = tx.write_set.iter().map(|v| *v.value()).collect();
+ drop(tx);
+ // Postprocessing: inserting row versions and logging the transaction to persistent storage.
+ // TODO: we should probably save to persistent storage first, and only then update the in-memory structures.
+ let mut log_record: LogRecord = LogRecord::new(end_ts);
+ for ref id in write_set {
+ if let Some(row_versions) = self.rows.get(id) {
+ let mut row_versions = row_versions.value().write().unwrap();
+ for row_version in row_versions.iter_mut() {
+ if let TxTimestampOrID::TxID(id) = row_version.begin {
+ if id == tx_id {
+ row_version.begin = TxTimestampOrID::Timestamp(tx_begin_ts);
+ self.insert_version_raw(
+ &mut log_record.row_versions,
+ row_version.clone(),
+ ); // FIXME: optimize cloning out
+ }
+ }
+ if let Some(TxTimestampOrID::TxID(id)) = row_version.end {
+ if id == tx_id {
+ row_version.end = Some(TxTimestampOrID::Timestamp(end_ts));
+ self.insert_version_raw(
+ &mut log_record.row_versions,
+ row_version.clone(),
+ ); // FIXME: optimize cloning out
+ }
+ }
+ }
+ }
+ }
+ tracing::trace!("UPDATED TX{tx_id}");
+ // We have now updated all the versions with a reference to the
+ // transaction ID to a timestamp and can, therefore, remove the
+ // transaction. Please note that when we move to lockless, the
+ // invariant doesn't necessarily hold anymore because another thread
+ // might have speculatively read a version that we want to remove.
+ // But that's a problem for another day.
+ // FIXME: it actually just become a problem for today!!!
+ // TODO: test that reproduces this failure, and then a fix
+ self.txs.remove(&tx_id);
+ if !log_record.row_versions.is_empty() {
+ self.storage.log_tx(log_record)?;
+ }
+ tracing::trace!("LOGGED {tx_id}");
+ Ok(())
+ }
+
+ /// Rolls back a transaction with the specified ID.
+ ///
+ /// This function rolls back a transaction with the specified `tx_id` by
+ /// discarding any changes made by the transaction.
+ ///
+ /// # Arguments
+ ///
+ /// * `tx_id` - The ID of the transaction to abort.
+ pub fn rollback_tx(&self, tx_id: TxID) {
+ let tx_unlocked = self.txs.get(&tx_id).unwrap();
+ let tx = tx_unlocked.value().write().unwrap();
+ assert_eq!(tx.state, TransactionState::Active);
+ tx.state.store(TransactionState::Aborted);
+ tracing::trace!("ABORT {tx}");
+ let write_set: Vec = tx.write_set.iter().map(|v| *v.value()).collect();
+ drop(tx);
+
+ for ref id in write_set {
+ if let Some(row_versions) = self.rows.get(id) {
+ let mut row_versions = row_versions.value().write().unwrap();
+ row_versions.retain(|rv| rv.begin != TxTimestampOrID::TxID(tx_id));
+ if row_versions.is_empty() {
+ self.rows.remove(id);
+ }
+ }
+ }
+
+ let tx = tx_unlocked.value().read().unwrap();
+ tx.state.store(TransactionState::Terminated);
+ tracing::trace!("TERMINATE {tx}");
+ // FIXME: verify that we can already remove the transaction here!
+ // Maybe it's fine for snapshot isolation, but too early for serializable?
+ self.txs.remove(&tx_id);
+ }
+
+ /// Generates next unique transaction id
+ pub fn get_tx_id(&self) -> u64 {
+ self.tx_ids.fetch_add(1, Ordering::SeqCst)
+ }
+
+ /// Gets current timestamp
+ pub fn get_timestamp(&self) -> u64 {
+ self.clock.get_timestamp()
+ }
+
+ /// Removes unused row versions with very loose heuristics,
+ /// which sometimes leaves versions intact for too long.
+ /// Returns the number of removed versions.
+ pub fn drop_unused_row_versions(&self) -> usize {
+ tracing::trace!(
+ "Dropping unused row versions. Database stats: transactions: {}; rows: {}",
+ self.txs.len(),
+ self.rows.len()
+ );
+ let mut dropped = 0;
+ let mut to_remove = Vec::new();
+ for entry in self.rows.iter() {
+ let mut row_versions = entry.value().write().unwrap();
+ row_versions.retain(|rv| {
+ // FIXME: should take rv.begin into account as well
+ let should_stay = match rv.end {
+ Some(TxTimestampOrID::Timestamp(version_end_ts)) => {
+ // a transaction started before this row version ended, ergo row version is needed
+ // NOTICE: O(row_versions x transactions), but also lock-free, so sounds acceptable
+ self.txs.iter().any(|tx| {
+ let tx = tx.value().read().unwrap();
+ // FIXME: verify!
+ match tx.state.load() {
+ TransactionState::Active | TransactionState::Preparing => {
+ version_end_ts > tx.begin_ts
+ }
+ _ => false,
+ }
+ })
+ }
+ // Let's skip potentially complex logic if the transafction is still
+ // active/tracked. We will drop the row version when the transaction
+ // gets garbage-collected itself, it will always happen eventually.
+ Some(TxTimestampOrID::TxID(tx_id)) => !self.txs.contains_key(&tx_id),
+ // this row version is current, ergo visible
+ None => true,
+ };
+ if !should_stay {
+ dropped += 1;
+ tracing::trace!(
+ "Dropping row version {:?} {:?}-{:?}",
+ entry.key(),
+ rv.begin,
+ rv.end
+ );
+ }
+ should_stay
+ });
+ if row_versions.is_empty() {
+ to_remove.push(*entry.key());
+ }
+ }
+ for id in to_remove {
+ self.rows.remove(&id);
+ }
+ dropped
+ }
+
+ pub fn recover(&self) -> Result<()> {
+ let tx_log = self.storage.read_tx_log()?;
+ for record in tx_log {
+ tracing::debug!("RECOVERING {:?}", record);
+ for version in record.row_versions {
+ self.insert_version(version.row.id, version);
+ }
+ self.clock.reset(record.tx_timestamp);
+ }
+ Ok(())
+ }
+}
+
+/// A write-write conflict happens when transaction T_m attempts to update a
+/// row version that is currently being updated by an active transaction T_n.
+pub(crate) fn is_write_write_conflict(
+ txs: &SkipMap>,
+ tx: &Transaction,
+ rv: &RowVersion,
+) -> bool {
+ match rv.end {
+ Some(TxTimestampOrID::TxID(rv_end)) => {
+ let te = txs.get(&rv_end).unwrap();
+ let te = te.value().read().unwrap();
+ match te.state.load() {
+ TransactionState::Active | TransactionState::Preparing => tx.tx_id != te.tx_id,
+ _ => false,
+ }
+ }
+ Some(TxTimestampOrID::Timestamp(_)) => false,
+ None => false,
+ }
+}
+
+pub(crate) fn is_version_visible(
+ txs: &SkipMap>,
+ tx: &Transaction,
+ rv: &RowVersion,
+) -> bool {
+ is_begin_visible(txs, tx, rv) && is_end_visible(txs, tx, rv)
+}
+
+fn is_begin_visible(
+ txs: &SkipMap>,
+ tx: &Transaction,
+ rv: &RowVersion,
+) -> bool {
+ match rv.begin {
+ TxTimestampOrID::Timestamp(rv_begin_ts) => tx.begin_ts >= rv_begin_ts,
+ TxTimestampOrID::TxID(rv_begin) => {
+ let tb = txs.get(&rv_begin).unwrap();
+ let tb = tb.value().read().unwrap();
+ let visible = match tb.state.load() {
+ TransactionState::Active => tx.tx_id == tb.tx_id && rv.end.is_none(),
+ TransactionState::Preparing => false, // NOTICE: makes sense for snapshot isolation, not so much for serializable!
+ TransactionState::Committed(committed_ts) => tx.begin_ts >= committed_ts,
+ TransactionState::Aborted => false,
+ TransactionState::Terminated => {
+ tracing::debug!("TODO: should reread rv's end field - it should have updated the timestamp in the row version by now");
+ false
+ }
+ };
+ tracing::trace!(
+ "is_begin_visible: tx={tx}, tb={tb} rv = {:?}-{:?} visible = {visible}",
+ rv.begin,
+ rv.end
+ );
+ visible
+ }
+ }
+}
+
+fn is_end_visible(
+ txs: &SkipMap>,
+ tx: &Transaction,
+ rv: &RowVersion,
+) -> bool {
+ match rv.end {
+ Some(TxTimestampOrID::Timestamp(rv_end_ts)) => tx.begin_ts < rv_end_ts,
+ Some(TxTimestampOrID::TxID(rv_end)) => {
+ let te = txs.get(&rv_end).unwrap();
+ let te = te.value().read().unwrap();
+ let visible = match te.state.load() {
+ TransactionState::Active => tx.tx_id != te.tx_id,
+ TransactionState::Preparing => false, // NOTICE: makes sense for snapshot isolation, not so much for serializable!
+ TransactionState::Committed(committed_ts) => tx.begin_ts < committed_ts,
+ TransactionState::Aborted => false,
+ TransactionState::Terminated => {
+ tracing::debug!("TODO: should reread rv's end field - it should have updated the timestamp in the row version by now");
+ false
+ }
+ };
+ tracing::trace!(
+ "is_end_visible: tx={tx}, te={te} rv = {:?}-{:?} visible = {visible}",
+ rv.begin,
+ rv.end
+ );
+ visible
+ }
+ None => true,
+ }
+}
diff --git a/core/mvcc/database/tests.rs b/core/mvcc/database/tests.rs
new file mode 100644
index 000000000..741ada4cb
--- /dev/null
+++ b/core/mvcc/database/tests.rs
@@ -0,0 +1,760 @@
+use super::*;
+use crate::mvcc::clock::LocalClock;
+
+fn test_db() -> Database {
+ let clock = LocalClock::new();
+ let storage = crate::mvcc::persistent_storage::Storage::new_noop();
+ Database::new(clock, storage)
+}
+
+#[test]
+fn test_insert_read() {
+ let db = test_db();
+
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+ db.commit_tx(tx1).unwrap();
+
+ let tx2 = db.begin_tx();
+ let row = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+}
+
+#[test]
+fn test_read_nonexistent() {
+ let db = test_db();
+ let tx = db.begin_tx();
+ let row = db.read(
+ tx,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ );
+ assert!(row.unwrap().is_none());
+}
+
+#[test]
+fn test_delete() {
+ let db = test_db();
+
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+ db.delete(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ assert!(row.is_none());
+ db.commit_tx(tx1).unwrap();
+
+ let tx2 = db.begin_tx();
+ let row = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ assert!(row.is_none());
+}
+
+#[test]
+fn test_delete_nonexistent() {
+ let db = test_db();
+ let tx = db.begin_tx();
+ assert!(!db
+ .delete(
+ tx,
+ RowID {
+ table_id: 1,
+ row_id: 1
+ }
+ )
+ .unwrap());
+}
+
+#[test]
+fn test_commit() {
+ let db = test_db();
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+ let tx1_updated_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ };
+ db.update(tx1, tx1_updated_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_updated_row, row);
+ db.commit_tx(tx1).unwrap();
+
+ let tx2 = db.begin_tx();
+ let row = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ db.commit_tx(tx2).unwrap();
+ assert_eq!(tx1_updated_row, row);
+ db.drop_unused_row_versions();
+}
+
+#[test]
+fn test_rollback() {
+ let db = test_db();
+ let tx1 = db.begin_tx();
+ let row1 = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, row1.clone()).unwrap();
+ let row2 = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(row1, row2);
+ let row3 = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ };
+ db.update(tx1, row3.clone()).unwrap();
+ let row4 = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(row3, row4);
+ db.rollback_tx(tx1);
+ let tx2 = db.begin_tx();
+ let row5 = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ assert_eq!(row5, None);
+}
+
+#[test]
+fn test_dirty_write() {
+ let db = test_db();
+
+ // T1 inserts a row with ID 1, but does not commit.
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+
+ // T2 attempts to delete row with ID 1, but fails because T1 has not committed.
+ let tx2 = db.begin_tx();
+ let tx2_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ };
+ assert!(!db.update(tx2, tx2_row).unwrap());
+
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+}
+
+#[test]
+fn test_dirty_read() {
+ let db = test_db();
+
+ // T1 inserts a row with ID 1, but does not commit.
+ let tx1 = db.begin_tx();
+ let row1 = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, row1).unwrap();
+
+ // T2 attempts to read row with ID 1, but doesn't see one because T1 has not committed.
+ let tx2 = db.begin_tx();
+ let row2 = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ assert_eq!(row2, None);
+}
+
+#[test]
+fn test_dirty_read_deleted() {
+ let db = test_db();
+
+ // T1 inserts a row with ID 1 and commits.
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ db.commit_tx(tx1).unwrap();
+
+ // T2 deletes row with ID 1, but does not commit.
+ let tx2 = db.begin_tx();
+ assert!(db
+ .delete(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1
+ }
+ )
+ .unwrap());
+
+ // T3 reads row with ID 1, but doesn't see the delete because T2 hasn't committed.
+ let tx3 = db.begin_tx();
+ let row = db
+ .read(
+ tx3,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+}
+
+#[test]
+fn test_fuzzy_read() {
+ let db = test_db();
+
+ // T1 inserts a row with ID 1 and commits.
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+ db.commit_tx(tx1).unwrap();
+
+ // T2 reads the row with ID 1 within an active transaction.
+ let tx2 = db.begin_tx();
+ let row = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+
+ // T3 updates the row and commits.
+ let tx3 = db.begin_tx();
+ let tx3_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ };
+ db.update(tx3, tx3_row).unwrap();
+ db.commit_tx(tx3).unwrap();
+
+ // T2 still reads the same version of the row as before.
+ let row = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+}
+
+#[test]
+fn test_lost_update() {
+ let db = test_db();
+
+ // T1 inserts a row with ID 1 and commits.
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+ db.commit_tx(tx1).unwrap();
+
+ // T2 attempts to update row ID 1 within an active transaction.
+ let tx2 = db.begin_tx();
+ let tx2_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "World".to_string(),
+ };
+ assert!(db.update(tx2, tx2_row.clone()).unwrap());
+
+ // T3 also attempts to update row ID 1 within an active transaction.
+ let tx3 = db.begin_tx();
+ let tx3_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "Hello, world!".to_string(),
+ };
+ assert_eq!(
+ Err(DatabaseError::WriteWriteConflict),
+ db.update(tx3, tx3_row)
+ );
+
+ db.commit_tx(tx2).unwrap();
+ assert_eq!(Err(DatabaseError::TxTerminated), db.commit_tx(tx3));
+
+ let tx4 = db.begin_tx();
+ let row = db
+ .read(
+ tx4,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx2_row, row);
+}
+
+// Test for the visibility to check if a new transaction can see old committed values.
+// This test checks for the typo present in the paper, explained in https://github.com/penberg/mvcc-rs/issues/15
+#[test]
+fn test_committed_visibility() {
+ let db = test_db();
+
+ // let's add $10 to my account since I like money
+ let tx1 = db.begin_tx();
+ let tx1_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "10".to_string(),
+ };
+ db.insert(tx1, tx1_row.clone()).unwrap();
+ db.commit_tx(tx1).unwrap();
+
+ // but I like more money, so let me try adding $10 more
+ let tx2 = db.begin_tx();
+ let tx2_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "20".to_string(),
+ };
+ assert!(db.update(tx2, tx2_row.clone()).unwrap());
+ let row = db
+ .read(
+ tx2,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(row, tx2_row);
+
+ // can I check how much money I have?
+ let tx3 = db.begin_tx();
+ let row = db
+ .read(
+ tx3,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap()
+ .unwrap();
+ assert_eq!(tx1_row, row);
+}
+
+// Test to check if a older transaction can see (un)committed future rows
+#[test]
+fn test_future_row() {
+ let db = test_db();
+
+ let tx1 = db.begin_tx();
+
+ let tx2 = db.begin_tx();
+ let tx2_row = Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "10".to_string(),
+ };
+ db.insert(tx2, tx2_row).unwrap();
+
+ // transaction in progress, so tx1 shouldn't be able to see the value
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ assert_eq!(row, None);
+
+ // lets commit the transaction and check if tx1 can see it
+ db.commit_tx(tx2).unwrap();
+ let row = db
+ .read(
+ tx1,
+ RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ )
+ .unwrap();
+ assert_eq!(row, None);
+}
+
+/* States described in the Hekaton paper *for serializability*:
+
+Table 1: Case analysis of action to take when version V’s
+Begin field contains the ID of transaction TB
+------------------------------------------------------------------------------------------------------
+TB’s state | TB’s end timestamp | Action to take when transaction T checks visibility of version V.
+------------------------------------------------------------------------------------------------------
+Active | Not set | V is visible only if TB=T and V’s end timestamp equals infinity.
+------------------------------------------------------------------------------------------------------
+Preparing | TS | V’s begin timestamp will be TS ut V is not yet committed. Use TS
+ | as V’s begin time when testing visibility. If the test is true,
+ | allow T to speculatively read V. Committed TS V’s begin timestamp
+ | will be TS and V is committed. Use TS as V’s begin time to test
+ | visibility.
+------------------------------------------------------------------------------------------------------
+Committed | TS | V’s begin timestamp will be TS and V is committed. Use TS as V’s
+ | begin time to test visibility.
+------------------------------------------------------------------------------------------------------
+Aborted | Irrelevant | Ignore V; it’s a garbage version.
+------------------------------------------------------------------------------------------------------
+Terminated | Irrelevant | Reread V’s Begin field. TB has terminated so it must have finalized
+or not found | | the timestamp.
+------------------------------------------------------------------------------------------------------
+
+Table 2: Case analysis of action to take when V's End field
+contains a transaction ID TE.
+------------------------------------------------------------------------------------------------------
+TE’s state | TE’s end timestamp | Action to take when transaction T checks visibility of a version V
+ | | as of read time RT.
+------------------------------------------------------------------------------------------------------
+Active | Not set | V is visible only if TE is not T.
+------------------------------------------------------------------------------------------------------
+Preparing | TS | V’s end timestamp will be TS provided that TE commits. If TS > RT,
+ | V is visible to T. If TS < RT, T speculatively ignores V.
+------------------------------------------------------------------------------------------------------
+Committed | TS | V’s end timestamp will be TS and V is committed. Use TS as V’s end
+ | timestamp when testing visibility.
+------------------------------------------------------------------------------------------------------
+Aborted | Irrelevant | V is visible.
+------------------------------------------------------------------------------------------------------
+Terminated | Irrelevant | Reread V’s End field. TE has terminated so it must have finalized
+or not found | | the timestamp.
+*/
+
+fn new_tx(tx_id: TxID, begin_ts: u64, state: TransactionState) -> RwLock {
+ let state = state.into();
+ RwLock::new(Transaction {
+ state,
+ tx_id,
+ begin_ts,
+ write_set: SkipSet::new(),
+ read_set: SkipSet::new(),
+ })
+}
+
+#[test]
+fn test_snapshot_isolation_tx_visible1() {
+ let txs: SkipMap> = SkipMap::from_iter([
+ (1, new_tx(1, 1, TransactionState::Committed(2))),
+ (2, new_tx(2, 2, TransactionState::Committed(5))),
+ (3, new_tx(3, 3, TransactionState::Aborted)),
+ (5, new_tx(5, 5, TransactionState::Preparing)),
+ (6, new_tx(6, 6, TransactionState::Committed(10))),
+ (7, new_tx(7, 7, TransactionState::Active)),
+ ]);
+
+ let current_tx = new_tx(4, 4, TransactionState::Preparing);
+ let current_tx = current_tx.read().unwrap();
+
+ let rv_visible = |begin: TxTimestampOrID, end: Option| {
+ let row_version = RowVersion {
+ begin,
+ end,
+ row: Row {
+ id: RowID {
+ table_id: 1,
+ row_id: 1,
+ },
+ data: "testme".to_string(),
+ },
+ };
+ tracing::debug!("Testing visibility of {row_version:?}");
+ is_version_visible(&txs, ¤t_tx, &row_version)
+ };
+
+ // begin visible: transaction committed with ts < current_tx.begin_ts
+ // end visible: inf
+ assert!(rv_visible(TxTimestampOrID::TxID(1), None));
+
+ // begin invisible: transaction committed with ts > current_tx.begin_ts
+ assert!(!rv_visible(TxTimestampOrID::TxID(2), None));
+
+ // begin invisible: transaction aborted
+ assert!(!rv_visible(TxTimestampOrID::TxID(3), None));
+
+ // begin visible: timestamp < current_tx.begin_ts
+ // end invisible: transaction committed with ts > current_tx.begin_ts
+ assert!(!rv_visible(
+ TxTimestampOrID::Timestamp(0),
+ Some(TxTimestampOrID::TxID(1))
+ ));
+
+ // begin visible: timestamp < current_tx.begin_ts
+ // end visible: transaction committed with ts < current_tx.begin_ts
+ assert!(rv_visible(
+ TxTimestampOrID::Timestamp(0),
+ Some(TxTimestampOrID::TxID(2))
+ ));
+
+ // begin visible: timestamp < current_tx.begin_ts
+ // end invisible: transaction aborted
+ assert!(!rv_visible(
+ TxTimestampOrID::Timestamp(0),
+ Some(TxTimestampOrID::TxID(3))
+ ));
+
+ // begin invisible: transaction preparing
+ assert!(!rv_visible(TxTimestampOrID::TxID(5), None));
+
+ // begin invisible: transaction committed with ts > current_tx.begin_ts
+ assert!(!rv_visible(TxTimestampOrID::TxID(6), None));
+
+ // begin invisible: transaction active
+ assert!(!rv_visible(TxTimestampOrID::TxID(7), None));
+
+ // begin invisible: transaction committed with ts > current_tx.begin_ts
+ assert!(!rv_visible(TxTimestampOrID::TxID(6), None));
+
+ // begin invisible: transaction active
+ assert!(!rv_visible(TxTimestampOrID::TxID(7), None));
+
+ // begin visible: timestamp < current_tx.begin_ts
+ // end invisible: transaction preparing
+ assert!(!rv_visible(
+ TxTimestampOrID::Timestamp(0),
+ Some(TxTimestampOrID::TxID(5))
+ ));
+
+ // begin invisible: timestamp > current_tx.begin_ts
+ assert!(!rv_visible(
+ TxTimestampOrID::Timestamp(6),
+ Some(TxTimestampOrID::TxID(6))
+ ));
+
+ // begin visible: timestamp < current_tx.begin_ts
+ // end visible: some active transaction will eventually overwrite this version,
+ // but that hasn't happened
+ // (this is the https://avi.im/blag/2023/hekaton-paper-typo/ case, I believe!)
+ assert!(rv_visible(
+ TxTimestampOrID::Timestamp(0),
+ Some(TxTimestampOrID::TxID(7))
+ ));
+}
diff --git a/core/mvcc/errors.rs b/core/mvcc/errors.rs
new file mode 100644
index 000000000..6cdad8ca3
--- /dev/null
+++ b/core/mvcc/errors.rs
@@ -0,0 +1,13 @@
+use thiserror::Error;
+
+#[derive(Error, Debug, PartialEq)]
+pub enum DatabaseError {
+ #[error("no such transaction ID: `{0}`")]
+ NoSuchTransactionID(u64),
+ #[error("transaction aborted because of a write-write conflict")]
+ WriteWriteConflict,
+ #[error("transaction is terminated")]
+ TxTerminated,
+ #[error("I/O error: {0}")]
+ Io(String),
+}
diff --git a/core/mvcc/mod.rs b/core/mvcc/mod.rs
new file mode 100644
index 000000000..53648ec2b
--- /dev/null
+++ b/core/mvcc/mod.rs
@@ -0,0 +1,160 @@
+//! Multiversion concurrency control (MVCC) for Rust.
+//!
+//! This module implements the main memory MVCC method outlined in the paper
+//! "High-Performance Concurrency Control Mechanisms for Main-Memory Databases"
+//! by Per-Åke Larson et al (VLDB, 2011).
+//!
+//! ## Data anomalies
+//!
+//! * A *dirty write* occurs when transaction T_m updates a value that is written by
+//! transaction T_n but not yet committed. The MVCC algorithm prevents dirty
+//! writes by validating that a row version is visible to transaction T_m before
+//! allowing update to it.
+//!
+//! * A *dirty read* occurs when transaction T_m reads a value that was written by
+//! transaction T_n but not yet committed. The MVCC algorithm prevents dirty
+//! reads by validating that a row version is visible to transaction T_m.
+//!
+//! * A *fuzzy read* (non-repeatable read) occurs when transaction T_m reads a
+//! different value in the course of the transaction because another
+//! transaction T_n has updated the value.
+//!
+//! * A *lost update* occurs when transactions T_m and T_n both attempt to update
+//! the same value, resulting in one of the updates being lost. The MVCC algorithm
+//! prevents lost updates by detecting the write-write conflict and letting the
+//! first-writer win by aborting the later transaction.
+//!
+//! TODO: phantom reads, cursor lost updates, read skew, write skew.
+//!
+//! ## TODO
+//!
+//! * Optimistic reads and writes
+//! * Garbage collection
+
+pub mod clock;
+pub mod cursor;
+pub mod database;
+pub mod errors;
+pub mod persistent_storage;
+
+#[cfg(test)]
+mod tests {
+ use crate::mvcc::clock::LocalClock;
+ use crate::mvcc::database::{Database, Row, RowID};
+ use std::sync::atomic::AtomicU64;
+ use std::sync::atomic::Ordering;
+ use std::sync::Arc;
+
+ static IDS: AtomicU64 = AtomicU64::new(1);
+
+ #[test]
+ fn test_non_overlapping_concurrent_inserts() {
+ // Two threads insert to the database concurrently using non-overlapping
+ // row IDs.
+ let clock = LocalClock::default();
+ let storage = crate::mvcc::persistent_storage::Storage::new_noop();
+ let db = Arc::new(Database::new(clock, storage));
+ let iterations = 100000;
+
+ let th1 = {
+ let db = db.clone();
+ std::thread::spawn(move || {
+ for _ in 0..iterations {
+ let tx = db.begin_tx();
+ let id = IDS.fetch_add(1, Ordering::SeqCst);
+ let id = RowID {
+ table_id: 1,
+ row_id: id,
+ };
+ let row = Row {
+ id,
+ data: "Hello".to_string(),
+ };
+ db.insert(tx, row.clone()).unwrap();
+ db.commit_tx(tx).unwrap();
+ let tx = db.begin_tx();
+ let committed_row = db.read(tx, id).unwrap();
+ db.commit_tx(tx).unwrap();
+ assert_eq!(committed_row, Some(row));
+ }
+ })
+ };
+ let th2 = {
+ std::thread::spawn(move || {
+ for _ in 0..iterations {
+ let tx = db.begin_tx();
+ let id = IDS.fetch_add(1, Ordering::SeqCst);
+ let id = RowID {
+ table_id: 1,
+ row_id: id,
+ };
+ let row = Row {
+ id,
+ data: "World".to_string(),
+ };
+ db.insert(tx, row.clone()).unwrap();
+ db.commit_tx(tx).unwrap();
+ let tx = db.begin_tx();
+ let committed_row = db.read(tx, id).unwrap();
+ db.commit_tx(tx).unwrap();
+ assert_eq!(committed_row, Some(row));
+ }
+ })
+ };
+ th1.join().unwrap();
+ th2.join().unwrap();
+ }
+
+ // FIXME: This test fails sporadically.
+ #[test]
+ #[ignore]
+ fn test_overlapping_concurrent_inserts_read_your_writes() {
+ let clock = LocalClock::default();
+ let storage = crate::mvcc::persistent_storage::Storage::new_noop();
+ let db = Arc::new(Database::new(clock, storage));
+ let iterations = 100000;
+
+ let work = |prefix: &'static str| {
+ let db = db.clone();
+ std::thread::spawn(move || {
+ let mut failed_upserts = 0;
+ for i in 0..iterations {
+ if i % 1000 == 0 {
+ tracing::debug!("{prefix}: {i}");
+ }
+ if i % 10000 == 0 {
+ let dropped = db.drop_unused_row_versions();
+ tracing::debug!("garbage collected {dropped} versions");
+ }
+ let tx = db.begin_tx();
+ let id = i % 16;
+ let id = RowID {
+ table_id: 1,
+ row_id: id,
+ };
+ let row = Row {
+ id,
+ data: format!("{prefix} @{tx}"),
+ };
+ if let Err(e) = db.upsert(tx, row.clone()) {
+ tracing::trace!("upsert failed: {e}");
+ failed_upserts += 1;
+ continue;
+ }
+ let committed_row = db.read(tx, id).unwrap();
+ db.commit_tx(tx).unwrap();
+ assert_eq!(committed_row, Some(row));
+ }
+ tracing::info!(
+ "{prefix}'s failed upserts: {failed_upserts}/{iterations} {:.2}%",
+ (failed_upserts * 100) as f64 / iterations as f64
+ );
+ })
+ };
+
+ let threads = vec![work("A"), work("B"), work("C"), work("D")];
+ for th in threads {
+ th.join().unwrap();
+ }
+ }
+}
diff --git a/core/mvcc/persistent_storage/mod.rs b/core/mvcc/persistent_storage/mod.rs
new file mode 100644
index 000000000..3f9ff2171
--- /dev/null
+++ b/core/mvcc/persistent_storage/mod.rs
@@ -0,0 +1,32 @@
+use std::fmt::Debug;
+
+use crate::mvcc::database::{LogRecord, Result};
+use crate::mvcc::errors::DatabaseError;
+
+#[derive(Debug)]
+pub enum Storage {
+ Noop,
+}
+
+impl Storage {
+ pub fn new_noop() -> Self {
+ Self::Noop
+ }
+}
+
+impl Storage {
+ pub fn log_tx(&self, _m: LogRecord
- 
