Merge 'Storage module cleanup' from Pekka Enberg

This pull request moves bunch of code under a new  module. The
idea is that we have three major subsystems:

- translate -- that performs code generation for SQL statements
- vdbe -- the runtime that executes SQL statements
- storage -- the storage layer that runtime uses

Closes #254

core/lib.rs

@@ -1,19 +1,14 @@
-mod btree;
-mod buffer_pool;
 mod datetime;
 mod error;
 mod function;
 mod io;
-mod pager;
 mod pseudo;
 mod schema;
-mod sqlite3_ondisk;
 mod storage;
 mod translate;
 mod types;
 mod util;
 mod vdbe;
-mod wal;
 
 #[cfg(not(target_family = "wasm"))]
 #[global_allocator]
@@ -21,12 +16,12 @@ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
 
 use fallible_iterator::FallibleIterator;
 use log::trace;
-use pager::Pager;
 use schema::Schema;
-use sqlite3_ondisk::DatabaseHeader;
 use sqlite3_parser::{ast::Cmd, lexer::sql::Parser};
 use std::sync::Arc;
 use std::{cell::RefCell, rc::Rc};
+use storage::pager::Pager;
+use storage::sqlite3_ondisk::DatabaseHeader;
 
 pub use error::LimboError;
 pub type Result<T> = std::result::Result<T, error::LimboError>;

core/storage/btree.rs

@@ -1,5 +1,5 @@
-use crate::pager::{Page, Pager};
-use crate::sqlite3_ondisk::{
+use crate::storage::pager::{Page, Pager};
+use crate::storage::sqlite3_ondisk::{
     read_varint, write_varint, BTreeCell, DatabaseHeader, PageContent, PageType, TableInteriorCell,
     TableLeafCell,
 };
@@ -124,7 +124,7 @@ impl BTreeCursor {
                     first_overflow_page: _,
                 }) => {
                     mem_page.advance();
-                    let record = crate::sqlite3_ondisk::read_record(_payload)?;
+                    let record = crate::storage::sqlite3_ondisk::read_record(_payload)?;
                     return Ok(CursorResult::Ok((Some(*_rowid), Some(record))));
                 }
                 BTreeCell::IndexInteriorCell(_) => {

core/storage/mod.rs

@@ -1,3 +1,9 @@
+pub(crate) mod btree;
+pub(crate) mod buffer_pool;
+pub(crate) mod pager;
+pub(crate) mod sqlite3_ondisk;
+pub(crate) mod wal;
+
 #[cfg(feature = "fs")]
 use crate::io::File;
 use crate::{error::LimboError, io::Completion, Buffer, Result};

core/storage/pager.rs

@@ -1,7 +1,6 @@
-use crate::buffer_pool::BufferPool;
-use crate::sqlite3_ondisk::PageContent;
-use crate::sqlite3_ondisk::{self, DatabaseHeader};
-use crate::wal::Wal;
+use crate::storage::buffer_pool::BufferPool;
+use crate::storage::sqlite3_ondisk::{self, DatabaseHeader, PageContent};
+use crate::storage::wal::Wal;
 use crate::{Buffer, PageSource, Result};
 use log::trace;
 use sieve_cache::SieveCache;

core/storage/sqlite3_ondisk.rs

@@ -41,10 +41,10 @@
 //!
 //! https://www.sqlite.org/fileformat.html
 
-use crate::buffer_pool::BufferPool;
 use crate::error::LimboError;
 use crate::io::{Buffer, Completion, ReadCompletion, WriteCompletion};
-use crate::pager::{Page, Pager};
+use crate::storage::buffer_pool::BufferPool;
+use crate::storage::pager::{Page, Pager};
 use crate::types::{OwnedRecord, OwnedValue};
 use crate::{File, PageSource, Result};
 use log::trace;

core/storage/wal.rs

@@ -1,6 +1,6 @@
 use std::{cell::RefCell, rc::Rc};
 
-use crate::{pager::Page, Result};
+use crate::{storage::pager::Page, Result};
 
 /// Write-ahead log (WAL).
 pub struct Wal {}

core/translate/insert.rs

@@ -7,7 +7,7 @@ use sqlite3_parser::ast::{
 use crate::Result;
 use crate::{
     schema::{Schema, Table},
-    sqlite3_ondisk::DatabaseHeader,
+    storage::sqlite3_ondisk::DatabaseHeader,
     translate::expr::translate_expr,
     vdbe::{builder::ProgramBuilder, Insn, Program},
 };

core/translate/mod.rs

@@ -15,9 +15,9 @@ pub(crate) mod where_clause;
 use std::cell::RefCell;
 use std::rc::Rc;
 
-use crate::pager::Pager;
 use crate::schema::Schema;
-use crate::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE};
+use crate::storage::pager::Pager;
+use crate::storage::sqlite3_ondisk::{DatabaseHeader, MIN_PAGE_CACHE_SIZE};
 use crate::util::normalize_ident;
 use crate::vdbe::{builder::ProgramBuilder, Insn, Program};
 use crate::{bail_parse_error, Result};

core/translate/select.rs

@@ -1,6 +1,6 @@
 use crate::function::{AggFunc, Func};
 use crate::schema::{Column, PseudoTable, Schema, Table};
-use crate::sqlite3_ondisk::DatabaseHeader;
+use crate::storage::sqlite3_ondisk::DatabaseHeader;
 use crate::translate::expr::{analyze_columns, maybe_apply_affinity, translate_expr};
 use crate::translate::where_clause::{
     process_where, translate_processed_where, translate_tableless_where, ProcessedWhereClause,

core/types.rs

@@ -4,7 +4,7 @@ use std::{cell::Ref, rc::Rc};
 use crate::error::LimboError;
 use crate::Result;
 
-use crate::sqlite3_ondisk::write_varint;
+use crate::storage::sqlite3_ondisk::write_varint;
 
 #[derive(Debug, Clone, PartialEq)]
 pub enum Value<'a> {

core/vdbe/builder.rs

@@ -1,6 +1,6 @@
 use std::{cell::RefCell, rc::Rc};
 
-use crate::sqlite3_ondisk::DatabaseHeader;
+use crate::storage::sqlite3_ondisk::DatabaseHeader;
 
 use super::{BranchOffset, CursorID, Insn, InsnReference, Program, Table};
 

core/vdbe/mod.rs

@@ -21,14 +21,13 @@ pub mod builder;
 pub mod explain;
 pub mod sorter;
 
-use crate::btree::BTreeCursor;
 use crate::datetime::{get_date_from_time_value, get_time_from_datetime_value};
 use crate::error::LimboError;
 use crate::function::{AggFunc, ScalarFunc};
-use crate::pager::Pager;
 use crate::pseudo::PseudoCursor;
 use crate::schema::Table;
-use crate::sqlite3_ondisk::DatabaseHeader;
+use crate::storage::sqlite3_ondisk::DatabaseHeader;
+use crate::storage::{btree::BTreeCursor, pager::Pager};
 use crate::types::{AggContext, Cursor, CursorResult, OwnedRecord, OwnedValue, Record};
 use crate::Result;