Merge 'Add .clone CLI command to copy database files' from Preston Thorpe

This PR adds support for the  `.clone file.db` sqlite3 CLI command

Closes #2437

bindings/javascript/src/lib.rs

@@ -516,4 +516,7 @@ impl turso_core::DatabaseStorage for DatabaseFile {
         let c = self.file.truncate(len, c)?;
         Ok(c)
     }
+    fn copy_to(&self, io: &dyn turso_core::IO, path: &str) -> turso_core::Result<()> {
+        self.file.copy_to(io, path)
+    }
 }

cli/app.rs

@@ -713,6 +713,11 @@ impl Limbo {
                         HeadersMode::Off => false,
                     };
                 }
+                Command::Clone(args) => {
+                    if let Err(e) = self.conn.copy_db(&args.output_file) {
+                        let _ = self.writeln(e.to_string());
+                    }
+                }
             },
         }
     }

cli/commands/args.rs

@@ -143,6 +143,11 @@ pub struct HeadersArgs {
     pub mode: HeadersMode,
 }
 
+#[derive(Debug, Clone, Args)]
+pub struct CloneArgs {
+    pub output_file: String,
+}
+
 #[derive(ValueEnum, Copy, Clone, Debug, PartialEq, Eq)]
 pub enum HeadersMode {
     On,

cli/commands/mod.rs

@@ -9,7 +9,10 @@ use args::{
 use clap::Parser;
 use import::ImportArgs;
 
-use crate::input::{AFTER_HELP_MSG, BEFORE_HELP_MSG};
+use crate::{
+    commands::args::CloneArgs,
+    input::{AFTER_HELP_MSG, BEFORE_HELP_MSG},
+};
 
 #[derive(Parser, Debug)]
 #[command(
@@ -86,6 +89,8 @@ pub enum Command {
     /// Toggle column headers on/off in list mode
     #[command(name = "headers", display_name = ".headers")]
     Headers(HeadersArgs),
+    #[command(name = "clone", display_name = ".clone")]
+    Clone(CloneArgs),
 }
 
 const _HELP_TEMPLATE: &str = "{before-help}{name}

cli/input.rs

@@ -182,13 +182,13 @@ pub fn get_io(db_location: DbLocation, io_choice: &str) -> anyhow::Result<Arc<dy
 
 pub const BEFORE_HELP_MSG: &str = r#"
 
-Limbo SQL Shell Help
+Turso SQL Shell Help
 ==============
-Welcome to the Limbo SQL Shell! You can execute any standard SQL command here.
+Welcome to the Turso SQL Shell! You can execute any standard SQL command here.
 In addition to standard SQL commands, the following special commands are available:"#;
 pub const AFTER_HELP_MSG: &str = r#"Usage Examples:
 ---------------
-1. To quit the Limbo SQL Shell:
+1. To quit the Turso SQL Shell:
    .quit
 
 2. To open a database file at path './employees.db':
@@ -239,6 +239,9 @@ pub const AFTER_HELP_MSG: &str = r#"Usage Examples:
 17. To turn off column headers in list mode:
    .headers off
 
+18. To clone the open database to another file:
+   .clone output_file.db
+
 Note:
 - All SQL commands must end with a semicolon (;).
 - Special commands start with a dot (.) and are not required to end with a semicolon."#;

core/io/mod.rs

@@ -52,6 +52,43 @@ pub trait File: Send + Sync {
     }
     fn size(&self) -> Result<u64>;
     fn truncate(&self, len: usize, c: Completion) -> Result<Completion>;
+    fn copy_to(&self, io: &dyn IO, path: &str) -> Result<()> {
+        // Open or create the destination file
+        let dest_file = io.open_file(path, OpenFlags::Create, false)?;
+        // Get the size of the source file
+        let file_size = self.size()? as usize;
+        if file_size == 0 {
+            return Ok(());
+        }
+
+        // use 1MB chunk size
+        const BUFFER_SIZE: usize = 1024 * 1024;
+        let mut pos = 0;
+
+        while pos < file_size {
+            let chunk_size = (file_size - pos).min(BUFFER_SIZE);
+            // Read from source
+            let read_buffer = Arc::new(Buffer::allocate(chunk_size, Rc::new(|_| {})));
+            let read_completion = self.pread(
+                pos,
+                Completion::new_read(read_buffer.clone(), move |_, _| {}),
+            )?;
+
+            // Wait for read to complete
+            io.wait_for_completion(read_completion)?;
+
+            // Write to destination
+            let write_completion =
+                dest_file.pwrite(pos, read_buffer, Completion::new_write(|_| {}))?;
+            io.wait_for_completion(write_completion)?;
+
+            pos += chunk_size;
+        }
+        let sync_completion = dest_file.sync(Completion::new_sync(|_| {}))?;
+        io.wait_for_completion(sync_completion)?;
+
+        Ok(())
+    }
 }
 
 #[derive(Debug, Copy, Clone, PartialEq)]

core/lib.rs

@@ -1737,6 +1737,24 @@ impl Connection {
     pub fn get_pager(&self) -> Rc<Pager> {
         self.pager.borrow().clone()
     }
+
+    #[cfg(feature = "fs")]
+    /// Copy the current Database and write out to a new file.
+    /// TODO: sqlite3 instead essentially does the equivalent of
+    /// `.dump` and creates a new .db file from that.
+    ///
+    /// Because we are instead making a copy of the File, as a side-effect we are
+    /// also having to checkpoint the database.
+    pub fn copy_db(&self, file: &str) -> Result<()> {
+        // use a new PlatformIO instance here to allow for copying in-memory databases
+        let io: Arc<dyn IO> = Arc::new(PlatformIO::new()?);
+        let disabled = false;
+        // checkpoint so everything is in the DB file before copying
+        self.pager
+            .borrow_mut()
+            .wal_checkpoint(disabled, CheckpointMode::Truncate)?;
+        self.pager.borrow_mut().db_file.copy_to(&*io, file)
+    }
 }
 
 pub struct Statement {

core/storage/database.rs

@@ -22,6 +22,7 @@ pub trait DatabaseStorage: Send + Sync {
     fn sync(&self, c: Completion) -> Result<Completion>;
     fn size(&self) -> Result<u64>;
     fn truncate(&self, len: usize, c: Completion) -> Result<Completion>;
+    fn copy_to(&self, io: &dyn crate::IO, path: &str) -> Result<()>;
 }
 
 #[cfg(feature = "fs")]
@@ -95,6 +96,11 @@ impl DatabaseStorage for DatabaseFile {
         let c = self.file.truncate(len, c)?;
         Ok(c)
     }
+
+    #[instrument(skip_all, level = Level::INFO)]
+    fn copy_to(&self, io: &dyn crate::IO, path: &str) -> Result<()> {
+        self.file.copy_to(io, path)
+    }
 }
 
 #[cfg(feature = "fs")]

testing/cli_tests/cli_test_cases.py

@@ -312,6 +312,49 @@ def test_uri_readonly():
     turso.quit()
 
 
+def test_copy_db_file():
+    testpath = "testing/test_copy.db"
+    if Path(testpath).exists():
+        os.unlink(Path(testpath))
+        time.sleep(0.2)  # make sure closed
+    time.sleep(0.3)
+    turso = TestTursoShell(init_commands="", flags=f" {testpath}")
+    turso.execute_dot("create table testing(a,b,c);")
+    turso.run_test_fn(".schema", lambda x: "CREATE TABLE testing (a, b, c)" in x, "test-database-has-expected-schema")
+    for i in range(100):
+        turso.execute_dot(f"insert into testing (a,b,c) values ({i},{i + 1}, {i + 2});")
+    turso.run_test_fn("SELECT COUNT(*) FROM testing;", lambda x: "100" == x, "test-database-has-expected-count")
+    turso.execute_dot(f".clone {testpath}")
+
+    turso.execute_dot(f".open {testpath}")
+    turso.run_test_fn(".schema", lambda x: "CREATE TABLE testing" in x, "test-copied-database-has-expected-schema")
+    turso.run_test_fn("SELECT COUNT(*) FROM testing;", lambda x: "100" == x, "test-copied-database-has-expected-count")
+    turso.quit()
+
+
+def test_copy_memory_db_to_file():
+    testpath = "testing/memory.db"
+    if Path(testpath).exists():
+        os.unlink(Path(testpath))
+        time.sleep(0.2)  # make sure closed
+
+    turso = TestTursoShell(init_commands="")
+    turso.execute_dot("create table testing(a,b,c);")
+    for i in range(100):
+        turso.execute_dot(f"insert into testing (a, b, c) values ({i},{i + 1}, {i + 2});")
+    turso.execute_dot(f".clone {testpath}")
+    turso.quit()
+    time.sleep(0.3)
+    sqlite = TestTursoShell(exec_name="sqlite3", flags=f" {testpath}")
+    sqlite.run_test_fn(
+        ".schema", lambda x: "CREATE TABLE testing (a, b, c)" in x, "test-copied-database-has-expected-schema"
+    )
+    sqlite.run_test_fn(
+        "SELECT COUNT(*) FROM testing;", lambda x: "100" == x, "test-copied-database-has-expected-user-count"
+    )
+    sqlite.quit()
+
+
 def main():
     console.info("Running all turso CLI tests...")
     test_basic_queries()
@@ -333,6 +376,8 @@ def main():
     test_update_with_limit()
     test_update_with_limit_and_offset()
     test_uri_readonly()
+    test_copy_db_file()
+    test_copy_memory_db_to_file()
     console.info("All tests have passed")
 
 

testing/cli_tests/test_turso_cli.py

@@ -135,9 +135,9 @@ INSERT INTO t VALUES (zeroblob(1024 - 1), zeroblob(1024 - 2), zeroblob(1024 - 3)
     def run_test(self, name: str, sql: str, expected: str) -> None:
         console.test(f"Running test: {name}", _stack_offset=2)
         actual = self.shell.execute(sql)
-        assert (
-            actual == expected
-        ), f"Test failed: {name}\nSQL: {sql}\nExpected:\n{repr(expected)}\nActual:\n{repr(actual)}"
+        assert actual == expected, (
+            f"Test failed: {name}\nSQL: {sql}\nExpected:\n{repr(expected)}\nActual:\n{repr(actual)}"
+        )
 
     def run_debug(self, sql: str):
         console.debug(f"debugging: {sql}", _stack_offset=2)
@@ -160,9 +160,10 @@ INSERT INTO t VALUES (zeroblob(1024 - 1), zeroblob(1024 - 2), zeroblob(1024 - 3)
         path = os.path.join("testing", "testing_clone.db")
         if os.path.exists(path):
             os.remove(path)
-        time.sleep(0.1)  # Ensure the file is removed before cloning
+        time.sleep(0.2)  # Ensure the file is removed before cloning
         cmd = "sqlite3 testing/testing.db '.clone testing/testing_clone.db'"
         subprocess.run(cmd, shell=True, capture_output=True, text=True)
+        time.sleep(0.2)  # Ensure lock releaesd
         if not os.path.exists("testing/testing_clone.db"):
             raise RuntimeError("Failed to clone testing.db to testing/testing_clone.db")