From 34c74e34d9ba04bcfd3630872bd733bbb4dcf40d Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Wed, 14 May 2025 12:24:14 +0200 Subject: [PATCH 1/5] test page_free_array Simply add a fuzz test to test free_array works as intended --- core/storage/btree.rs | 142 ++++++++++++++++++++++++++++++++++++++---- 1 file changed, 129 insertions(+), 13 deletions(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index bc7c4afe2..6c583f307 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -4622,12 +4622,7 @@ fn edit_page( usable_space, )?; // shift pointers left - let buf = page.as_ptr(); - let (start, _) = page.cell_pointer_array_offset_and_size(); - buf.copy_within( - start + (number_to_shift * 2)..start + (count_cells * 2), - start, - ); + shift_cells_left(page, count_cells, number_to_shift); count_cells -= number_to_shift; debug_validate_cells!(page, usable_space); } @@ -4688,6 +4683,15 @@ fn edit_page( Ok(()) } +fn shift_cells_left(page: &mut PageContent, count_cells: usize, number_to_shift: usize) { + let buf = page.as_ptr(); + let (start, _) = page.cell_pointer_array_offset_and_size(); + buf.copy_within( + start + (number_to_shift * 2)..start + (count_cells * 2), + start, + ); +} + fn page_free_array( page: &mut PageContent, first: usize, @@ -5766,11 +5770,14 @@ mod tests { } } - fn rng_from_time() -> (ChaCha8Rng, u64) { - let seed = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); + fn rng_from_time_or_env() -> (ChaCha8Rng, u64) { + let seed = std::env::var("SEED").map_or( + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(), + |v| v.parse().unwrap(), + ); let rng = ChaCha8Rng::seed_from_u64(seed); (rng, seed) } @@ -5780,7 +5787,7 @@ mod tests { inserts: usize, size: impl Fn(&mut ChaCha8Rng) -> usize, ) { - let (mut rng, seed) = rng_from_time(); + let (mut rng, seed) = rng_from_time_or_env(); let mut seen = HashSet::new(); tracing::info!("super seed: {}", seed); for _ in 0..attempts { @@ -5879,7 +5886,7 @@ mod tests { let rng = ChaCha8Rng::seed_from_u64(seed); (rng, seed) } else { - rng_from_time() + rng_from_time_or_env() }; let mut seen = HashSet::new(); tracing::info!("super seed: {}", seed); @@ -7108,4 +7115,113 @@ mod tests { } } } + + #[test] + fn test_free_array() { + let (mut rng, seed) = rng_from_time_or_env(); + tracing::info!("seed={}", seed); + + const ITERATIONS: usize = 10000; + for _ in 0..ITERATIONS { + let mut cell_array = CellArray { + cells: Vec::new(), + number_of_cells_per_page: [0; 5], + }; + let mut cells_cloned = Vec::new(); + let (pager, _) = empty_btree(); + let page_type = PageType::TableLeaf; + let page = pager.allocate_page().unwrap(); + btree_init_page(&page, page_type, 0, pager.usable_space() as u16); + let mut size = (rng.next_u64() % 100) as u16; + let mut i = 0; + // add a bunch of cells + while compute_free_space(page.get_contents(), pager.usable_space() as u16) >= size + 10 + { + insert_cell(i, size, page.get_contents(), pager.clone(), page_type); + i += 1; + size = (rng.next_u64() % 1024) as u16; + } + + // Create cell array with references to cells inserted + let contents = page.get_contents(); + for cell_idx in 0..contents.cell_count() { + let buf = contents.as_ptr(); + let (start, len) = contents.cell_get_raw_region( + cell_idx, + payload_overflow_threshold_max(contents.page_type(), 4096), + payload_overflow_threshold_min(contents.page_type(), 4096), + pager.usable_space(), + ); + cell_array + .cells + .push(to_static_buf(&mut buf[start..start + len])); + cells_cloned.push(buf[start..start + len].to_vec()); + } + + debug_validate_cells!(contents, pager.usable_space() as u16); + + // now free a prefix or suffix of cells added + let cells_before_free = contents.cell_count(); + let size = rng.next_u64() as usize % cells_before_free; + let prefix = rng.next_u64() % 2 == 0; + let start = if prefix { + 0 + } else { + contents.cell_count() - size + }; + let removed = page_free_array( + contents, + start, + size as usize, + &cell_array, + pager.usable_space() as u16, + ) + .unwrap(); + // shift if needed + if prefix { + shift_cells_left(contents, cells_before_free, removed); + } + + assert_eq!(removed, size); + assert_eq!(contents.cell_count(), cells_before_free - size); + debug_validate_cells_core(contents, pager.usable_space() as u16); + // check cells are correct + let mut cell_idx_cloned = if prefix { size } else { 0 }; + for cell_idx in 0..contents.cell_count() { + let buf = contents.as_ptr(); + let (start, len) = contents.cell_get_raw_region( + cell_idx, + payload_overflow_threshold_max(contents.page_type(), 4096), + payload_overflow_threshold_min(contents.page_type(), 4096), + pager.usable_space(), + ); + let cell_in_page = &buf[start..start + len]; + let cell_in_array = &cells_cloned[cell_idx_cloned]; + assert_eq!(cell_in_page, cell_in_array); + cell_idx_cloned += 1; + } + } + } + + fn insert_cell( + i: u64, + size: u16, + contents: &mut PageContent, + pager: Rc, + page_type: PageType, + ) { + let mut payload = Vec::new(); + let record = ImmutableRecord::from_registers(&[Register::OwnedValue(OwnedValue::Blob( + vec![0; size as usize], + ))]); + fill_cell_payload( + page_type, + Some(i), + &mut payload, + &record, + pager.usable_space() as u16, + pager.clone(), + ); + insert_into_cell(contents, &payload, i as usize, pager.usable_space() as u16).unwrap(); + } } From 0d8a94d49d3240e4351064d43e2169420339617b Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou <30913090+pereman2@users.noreply.github.com> Date: Wed, 14 May 2025 13:19:20 +0200 Subject: [PATCH 2/5] expect parse seed in rng_from_time_or_env Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- core/storage/btree.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 6c583f307..a6afa8223 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5776,7 +5776,7 @@ mod tests { .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(), - |v| v.parse().unwrap(), + |v| v.parse().expect("Failed to parse SEED environment variable as u64"), ); let rng = ChaCha8Rng::seed_from_u64(seed); (rng, seed) From 13a120530edbc98b87631423944e62997eecb464 Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou <30913090+pereman2@users.noreply.github.com> Date: Wed, 14 May 2025 13:20:11 +0200 Subject: [PATCH 3/5] comment shift_cells_left Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- core/storage/btree.rs | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index a6afa8223..4d4ecec4c 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -4683,6 +4683,17 @@ fn edit_page( Ok(()) } +/// Shifts the cell pointers in the B-tree page to the left by a specified number of positions. +/// +/// # Parameters +/// - `page`: A mutable reference to the `PageContent` representing the B-tree page. +/// - `count_cells`: The total number of cells currently in the page. +/// - `number_to_shift`: The number of cell pointers to shift to the left. +/// +/// # Behavior +/// This function modifies the cell pointer array within the page by copying memory regions. +/// It shifts the pointers starting from `number_to_shift` to the beginning of the array, +/// effectively removing the first `number_to_shift` pointers. fn shift_cells_left(page: &mut PageContent, count_cells: usize, number_to_shift: usize) { let buf = page.as_ptr(); let (start, _) = page.cell_pointer_array_offset_and_size(); From c3432e1cc31921ee6ea4c9c796407ee666eb87ba Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Wed, 14 May 2025 13:23:12 +0200 Subject: [PATCH 4/5] fmt --- core/storage/btree.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 4d4ecec4c..126c62996 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -5787,7 +5787,10 @@ mod tests { .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_secs(), - |v| v.parse().expect("Failed to parse SEED environment variable as u64"), + |v| { + v.parse() + .expect("Failed to parse SEED environment variable as u64") + }, ); let rng = ChaCha8Rng::seed_from_u64(seed); (rng, seed) From 5025655f757978379614d54386b7cb4b9f646afb Mon Sep 17 00:00:00 2001 From: Pere Diaz Bou Date: Wed, 14 May 2025 13:53:51 +0200 Subject: [PATCH 5/5] debug assertions --- core/storage/btree.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/storage/btree.rs b/core/storage/btree.rs index 126c62996..3aef65407 100644 --- a/core/storage/btree.rs +++ b/core/storage/btree.rs @@ -7198,6 +7198,7 @@ mod tests { assert_eq!(removed, size); assert_eq!(contents.cell_count(), cells_before_free - size); + #[cfg(debug_assertions)] debug_validate_cells_core(contents, pager.usable_space() as u16); // check cells are correct let mut cell_idx_cloned = if prefix { size } else { 0 };