make read_record, read_varint and read_value faster

We make read_record faster by not allocating Vec if not needed. This is
why I introduced a simple `SmallVec<T>` that will have a stack allocated
list for the simplest workloads, and a heap allocated if we were to
require more stuff.

Both read_varint and read_value, at least in my mac m4, were not
inlined. Since these functions are called so many times it made sense to
inline them to avoid call overhead. With this I saw something like 20%
improvement over previous commit in my m4.

core/storage/sqlite3_ondisk.rs

@@ -51,6 +51,7 @@ use crate::types::{ImmutableRecord, RawSlice, RefValue, TextRef, TextSubtype};
 use crate::{File, Result};
 use parking_lot::RwLock;
 use std::cell::RefCell;
+use std::mem::MaybeUninit;
 use std::pin::Pin;
 use std::rc::Rc;
 use std::sync::Arc;
@@ -1053,17 +1054,56 @@ pub fn validate_serial_type(value: u64) -> Result<SerialType> {
     }
 }
 
-pub fn read_record(payload: &[u8]) -> Result<ImmutableRecord> {
+struct SmallVec<T> {
+    pub data: [std::mem::MaybeUninit<T>; 64],
+    pub len: usize,
+    pub extra_data: Option<Vec<T>>,
+}
+
+impl<T: Default + Copy> SmallVec<T> {
+    pub fn new() -> Self {
+        Self {
+            data: unsafe { std::mem::MaybeUninit::uninit().assume_init() },
+            len: 0,
+            extra_data: None,
+        }
+    }
+
+    pub fn push(&mut self, value: T) {
+        if self.len < self.data.len() {
+            self.data[self.len] = MaybeUninit::new(value);
+            self.len += 1;
+        } else {
+            if self.extra_data.is_none() {
+                self.extra_data = Some(Vec::new());
+            }
+            self.extra_data.as_mut().unwrap().push(value);
+            self.len += 1;
+        }
+    }
+
+    pub fn has_extra_data(&self) -> bool {
+        self.len >= self.len
+    }
+}
+
+pub fn read_record(payload: &[u8], reuse_immutable: &mut ImmutableRecord) -> Result<()> {
+    // Let's clear previous use
+    reuse_immutable.payload.clear();
+    reuse_immutable.values.clear();
+    // Copy payload to ImmutableRecord in order to make RefValue that point to this new buffer.
+    // By reusing this immutable record we make it less allocation expensive.
+    reuse_immutable.payload.extend_from_slice(payload);
+
     let mut pos = 0;
     let (header_size, nr) = read_varint(payload)?;
     assert!((header_size as usize) >= nr);
     let mut header_size = (header_size as usize) - nr;
-    let payload = payload.to_vec();
     pos += nr;
 
-    let mut serial_types = Vec::with_capacity(header_size);
+    let mut serial_types = SmallVec::new();
     while header_size > 0 {
-        let (serial_type, nr) = read_varint(&payload[pos..])?;
+        let (serial_type, nr) = read_varint(&reuse_immutable.payload[pos..])?;
         let serial_type = validate_serial_type(serial_type)?;
         serial_types.push(serial_type);
         pos += nr;
@@ -1071,21 +1111,27 @@ pub fn read_record(payload: &[u8]) -> Result<ImmutableRecord> {
         header_size -= nr;
     }
 
-    let mut values = Vec::with_capacity(serial_types.len());
-    for &serial_type in &serial_types {
-        let (value, n) = read_value(&payload[pos..], serial_type)?;
+    for &serial_type in &serial_types.data[..serial_types.len.min(serial_types.data.len())] {
+        let (value, n) = read_value(&reuse_immutable.payload[pos..], unsafe {
+            *serial_type.as_ptr()
+        })?;
         pos += n;
-        values.push(value);
+        reuse_immutable.values.push(value);
+    }
+    if let Some(extra) = serial_types.extra_data.as_ref() {
+        for serial_type in extra {
+            let (value, n) = read_value(&reuse_immutable.payload[pos..], *serial_type)?;
+            pos += n;
+            reuse_immutable.values.push(value);
+        }
     }
 
-    Ok(ImmutableRecord {
-        payload: payload,
-        values,
-    })
+    Ok(())
 }
 
 /// Reads a value that might reference the buffer it is reading from. Be sure to store RefValue with the buffer
 /// always.
+#[inline(always)]
 pub fn read_value(buf: &[u8], serial_type: SerialType) -> Result<(RefValue, usize)> {
     if serial_type.is_null() {
         return Ok((RefValue::Null, 0));
@@ -1223,6 +1269,7 @@ pub fn read_value(buf: &[u8], serial_type: SerialType) -> Result<(RefValue, usiz
     crate::bail_corrupt_error!("Invalid serial type: {}", serial_type)
 }
 
+#[inline(always)]
 pub fn read_varint(buf: &[u8]) -> Result<(u64, usize)> {
     let mut v: u64 = 0;
     for i in 0..8 {