Merge 'Improve debug build validation speed' from Pere Diaz Bou

Various things to improve speed of long fuzz test execution time:
* remove unnecessary debug_validate_cell calls
* Add SortedVec for keys in fuzz tests
* Validate btree's depth in fuzz test every 1K inserts to not overload
test with validations. We add `VALIDATE_BTREE`  env variable to enable
validation on every insert in case it is needed.

Reviewed-by: Jussi Saurio <jussi.saurio@gmail.com>

Closes #1521

Cargo.lock

@@ -1839,6 +1839,7 @@ dependencies = [
  "rusqlite",
  "rustix 1.0.7",
  "ryu",
+ "sorted-vec",
  "strum",
  "tempfile",
  "test-log",
@@ -3301,6 +3302,12 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
+[[package]]
+name = "sorted-vec"
+version = "0.8.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d372029cb5195f9ab4e4b9aef550787dce78b124fcaee8d82519925defcd6f0d"
+
 [[package]]
 name = "sqlparser_bench"
 version = "0.1.0"

core/Cargo.toml

@@ -35,7 +35,7 @@ io-uring = { version = "0.7.5", optional = true }
 
 [target.'cfg(target_family = "unix")'.dependencies]
 polling = "3.7.4"
-rustix = { version = "1.0.5", features = ["fs"]}
+rustix = { version = "1.0.5", features = ["fs"] }
 
 [target.'cfg(not(target_family = "wasm"))'.dependencies]
 mimalloc = { version = "0.1.46", default-features = false }
@@ -99,6 +99,7 @@ rand_chacha = "0.9.0"
 env_logger = "0.11.6"
 test-log = { version = "0.2.17", features = ["trace"] }
 lru = "0.14.0"
+sorted-vec = "0.8.6"
 
 [[bench]]
 name = "benchmark"

core/storage/btree.rs

@@ -5230,11 +5230,10 @@ fn page_insert_array(
         page.page_type()
     );
     for i in first..first + count {
-        debug_validate_cells!(page, usable_space);
         insert_into_cell(page, cell_array.cells[i], start_insert, usable_space)?;
-        debug_validate_cells!(page, usable_space);
         start_insert += 1;
     }
+    debug_validate_cells!(page, usable_space);
     Ok(())
 }
 
@@ -5443,7 +5442,6 @@ fn insert_into_cell(
     cell_idx: usize,
     usable_space: u16,
 ) -> Result<()> {
-    debug_validate_cells!(page, usable_space);
     assert!(
         cell_idx <= page.cell_count() + page.overflow_cells.len(),
         "attempting to add cell to an incorrect place cell_idx={} cell_count={}",
@@ -5744,7 +5742,6 @@ fn payload_overflow_threshold_min(_page_type: PageType, usable_space: u16) -> us
 /// Drop a cell from a page.
 /// This is done by freeing the range of bytes that the cell occupies.
 fn drop_cell(page: &mut PageContent, cell_idx: usize, usable_space: u16) -> Result<()> {
-    debug_validate_cells!(page, usable_space);
     let (cell_start, cell_len) = page.cell_get_raw_region(
         cell_idx,
         payload_overflow_threshold_max(page.page_type(), usable_space),
@@ -5784,6 +5781,7 @@ mod tests {
         rand_core::{RngCore, SeedableRng},
         ChaCha8Rng,
     };
+    use sorted_vec::SortedVec;
     use test_log::test;
 
     use super::*;
@@ -6226,15 +6224,19 @@ mod tests {
         inserts: usize,
         size: impl Fn(&mut ChaCha8Rng) -> usize,
     ) {
+        const VALIDATE_INTERVAL: usize = 1000;
+        let do_validate_btree = std::env::var("VALIDATE_BTREE")
+            .map_or(false, |v| v.parse().expect("validate should be bool"));
         let (mut rng, seed) = rng_from_time_or_env();
         let mut seen = HashSet::new();
         tracing::info!("super seed: {}", seed);
         for _ in 0..attempts {
             let (pager, root_page) = empty_btree();
             let mut cursor = BTreeCursor::new(None, pager.clone(), root_page);
-            let mut keys = Vec::new();
+            let mut keys = SortedVec::new();
             tracing::info!("seed: {}", seed);
             for insert_id in 0..inserts {
+                let do_validate = do_validate_btree || (insert_id % VALIDATE_INTERVAL == 0);
                 let size = size(&mut rng);
                 let key = {
                     let result;
@@ -6267,28 +6269,34 @@ mod tests {
                 .unwrap();
                 let value =
                     ImmutableRecord::from_registers(&[Register::Value(Value::Blob(vec![0; size]))]);
-                let btree_before = format_btree(pager.clone(), root_page, 0);
+                let btree_before = if do_validate {
+                    format_btree(pager.clone(), root_page, 0)
+                } else {
+                    "".to_string()
+                };
                 run_until_done(
                     || cursor.insert(&BTreeKey::new_table_rowid(key as u64, Some(&value)), true),
                     pager.deref(),
                 )
                 .unwrap();
-                // FIXME: add sorted vector instead, should be okay for small amounts of keys for now :P, too lazy to fix right now
-                keys.sort();
-                cursor.move_to_root();
                 let mut valid = true;
-                for key in keys.iter() {
-                    tracing::trace!("seeking key: {}", key);
-                    run_until_done(|| cursor.next(), pager.deref()).unwrap();
-                    let cursor_rowid = cursor.rowid().unwrap().unwrap();
-                    if *key as u64 != cursor_rowid {
-                        valid = false;
-                        println!("key {} is not found, got {}", key, cursor_rowid);
-                        break;
+                if do_validate {
+                    cursor.move_to_root();
+                    for key in keys.iter() {
+                        tracing::trace!("seeking key: {}", key);
+                        run_until_done(|| cursor.next(), pager.deref()).unwrap();
+                        let cursor_rowid = cursor.rowid().unwrap().unwrap();
+                        if *key as u64 != cursor_rowid {
+                            valid = false;
+                            println!("key {} is not found, got {}", key, cursor_rowid);
+                            break;
+                        }
                     }
                 }
                 // let's validate btree too so that we undertsand where the btree failed
-                if matches!(validate_btree(pager.clone(), root_page), (_, false)) || !valid {
+                if do_validate
+                    && (!valid || matches!(validate_btree(pager.clone(), root_page), (_, false)))
+                {
                     let btree_after = format_btree(pager.clone(), root_page, 0);
                     println!("btree before:\n{}", btree_before);
                     println!("btree after:\n{}", btree_after);
@@ -6302,7 +6310,6 @@ mod tests {
             if matches!(validate_btree(pager.clone(), root_page), (_, false)) {
                 panic!("invalid btree");
             }
-            keys.sort();
             cursor.move_to_root();
             for key in keys.iter() {
                 tracing::trace!("seeking key: {}", key);
@@ -6333,7 +6340,7 @@ mod tests {
             let index_root_page = pager.btree_create(&CreateBTreeFlags::new_index());
             let index_root_page = index_root_page as usize;
             let mut cursor = BTreeCursor::new(None, pager.clone(), index_root_page);
-            let mut keys = Vec::new();
+            let mut keys = SortedVec::new();
             tracing::info!("seed: {}", seed);
             for _ in 0..inserts {
                 let key = {
@@ -6368,10 +6375,8 @@ mod tests {
                     pager.deref(),
                 )
                 .unwrap();
-                keys.sort();
                 cursor.move_to_root();
             }
-            keys.sort();
             cursor.move_to_root();
             for key in keys.iter() {
                 tracing::trace!("seeking key: {:?}", key);
@@ -6486,7 +6491,7 @@ mod tests {
     #[test]
     #[ignore]
     pub fn fuzz_long_btree_insert_fuzz_run_random() {
-        btree_insert_fuzz_run(128, 10_000, |rng| (rng.next_u32() % 4096) as usize);
+        btree_insert_fuzz_run(128, 2_000, |rng| (rng.next_u32() % 4096) as usize);
     }
 
     #[test]
@@ -6498,15 +6503,13 @@ mod tests {
     #[test]
     #[ignore]
     pub fn fuzz_long_btree_insert_fuzz_run_big() {
-        btree_insert_fuzz_run(64, 10_000, |rng| {
-            3 * 1024 + (rng.next_u32() % 1024) as usize
-        });
+        btree_insert_fuzz_run(64, 2_000, |rng| 3 * 1024 + (rng.next_u32() % 1024) as usize);
     }
 
     #[test]
     #[ignore]
     pub fn fuzz_long_btree_insert_fuzz_run_overflow() {
-        btree_insert_fuzz_run(64, 10_000, |rng| (rng.next_u32() % 32 * 1024) as usize);
+        btree_insert_fuzz_run(64, 5_000, |rng| (rng.next_u32() % 32 * 1024) as usize);
     }
 
     #[allow(clippy::arc_with_non_send_sync)]