aljaz/turso
1
0
Fork 0
mirror of https://github.com/aljazceru/turso.git synced 2025-12-19 09:34:18 +01:00
Code Issues Packages Projects Releases Wiki Activity

Merge 'Various JSON improvements' from Ihor Andrianov

Browse Source 
Added jsonb_object, jsonb_array, json_insert, jsonb_insert.
MongoDB is sweating now.

Closes #1160
This commit is contained in:
Pekka Enberg
2025-03-24 09:17:40 +02:00
parent 2bbc2b15bd b7be728d64
commit e8c0a6e728
7 changed files with 1762 additions and 922 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
COMPAT.md
View File

@@ -363,7 +363,7 @@ Modifiers:
| json(json) | Yes | |
| jsonb(json) | Yes | |
| json_array(value1,value2,...) | Yes | |
| jsonb_array(value1,value2,...) | | |
| jsonb_array(value1,value2,...) | Yes | |
| json_array_length(json) | Yes | |
| json_array_length(json,path) | Yes | |
| json_error_position(json) | Yes | |
@@ -371,10 +371,10 @@ Modifiers:
| jsonb_extract(json,path,...) | Yes | |
| json -> path | Yes | |
| json ->> path | Yes | |
| json_insert(json,path,value,...) | | |
| jsonb_insert(json,path,value,...) | | |
| json_object(label1,value1,...) | Yes | When keys are duplicated, only the last one processed is returned. This differs from sqlite, where the keys in the output can be duplicated |
| jsonb_object(label1,value1,...) | | |
| json_insert(json,path,value,...) | Yes | |
| jsonb_insert(json,path,value,...) | Yes | |
| json_object(label1,value1,...) | Yes | |
| jsonb_object(label1,value1,...) | Yes | |
| json_patch(json1,json2) | Yes | |
| jsonb_patch(json1,json2) | | |
| json_pretty(json) | Partial | Shares same json(val) limitations. Also, when passing blobs for indentation, conversion is not exactly the same as in SQLite |
@@ -383,7 +383,7 @@ Modifiers:
| json_replace(json,path,value,...) | Yes | |
| jsonb_replace(json,path,value,...) | Yes | |
| json_set(json,path,value,...) | Yes | |
| jsonb_set(json,path,value,...) | | |
| jsonb_set(json,path,value,...) | Yes | |
| json_type(json) | Yes | |
| json_type(json,path) | Yes | |
| json_valid(json) | Yes | |

16
core/function.rs
View File

@@ -73,12 +73,14 @@ pub enum JsonFunc {
Json,
Jsonb,
JsonArray,
JsonbArray,
JsonArrayLength,
JsonArrowExtract,
JsonArrowShiftExtract,
JsonExtract,
JsonbExtract,
JsonObject,
JsonbObject,
JsonType,
JsonErrorPosition,
JsonValid,
@@ -87,6 +89,8 @@ pub enum JsonFunc {
JsonbRemove,
JsonReplace,
JsonbReplace,
JsonInsert,
JsonbInsert,
JsonPretty,
JsonSet,
JsonQuote,
@@ -102,12 +106,14 @@ impl Display for JsonFunc {
Self::Json => "json".to_string(),
Self::Jsonb => "jsonb".to_string(),
Self::JsonArray => "json_array".to_string(),
Self::JsonbArray => "jsonb_array".to_string(),
Self::JsonExtract => "json_extract".to_string(),
Self::JsonbExtract => "jsonb_extract".to_string(),
Self::JsonArrayLength => "json_array_length".to_string(),
Self::JsonArrowExtract => "->".to_string(),
Self::JsonArrowShiftExtract => "->>".to_string(),
Self::JsonObject => "json_object".to_string(),
Self::JsonbObject => "jsonb_object".to_string(),
Self::JsonType => "json_type".to_string(),
Self::JsonErrorPosition => "json_error_position".to_string(),
Self::JsonValid => "json_valid".to_string(),
@@ -116,6 +122,8 @@ impl Display for JsonFunc {
Self::JsonbRemove => "jsonb_remove".to_string(),
Self::JsonReplace => "json_replace".to_string(),
Self::JsonbReplace => "jsonb_replace".to_string(),
Self::JsonInsert => "json_insert".to_string(),
Self::JsonbInsert => "jsonb_insert".to_string(),
Self::JsonPretty => "json_pretty".to_string(),
Self::JsonSet => "json_set".to_string(),
Self::JsonQuote => "json_quote".to_string(),
@@ -565,12 +573,16 @@ impl Func {
#[cfg(feature = "json")]
"json_array" => Ok(Self::Json(JsonFunc::JsonArray)),
#[cfg(feature = "json")]
"jsonb_array" => Ok(Self::Json(JsonFunc::JsonbArray)),
#[cfg(feature = "json")]
"json_extract" => Ok(Func::Json(JsonFunc::JsonExtract)),
#[cfg(feature = "json")]
"jsonb_extract" => Ok(Func::Json(JsonFunc::JsonbExtract)),
#[cfg(feature = "json")]
"json_object" => Ok(Func::Json(JsonFunc::JsonObject)),
#[cfg(feature = "json")]
"jsonb_object" => Ok(Func::Json(JsonFunc::JsonbObject)),
#[cfg(feature = "json")]
"json_type" => Ok(Func::Json(JsonFunc::JsonType)),
#[cfg(feature = "json")]
"json_error_position" => Ok(Self::Json(JsonFunc::JsonErrorPosition)),
@@ -585,6 +597,10 @@ impl Func {
#[cfg(feature = "json")]
"json_replace" => Ok(Self::Json(JsonFunc::JsonReplace)),
#[cfg(feature = "json")]
"json_insert" => Ok(Self::Json(JsonFunc::JsonInsert)),
#[cfg(feature = "json")]
"jsonb_insert" => Ok(Self::Json(JsonFunc::JsonbInsert)),
#[cfg(feature = "json")]
"jsonb_replace" => Ok(Self::Json(JsonFunc::JsonReplace)),
#[cfg(feature = "json")]
"json_pretty" => Ok(Self::Json(JsonFunc::JsonPretty)),

80
core/json/json_operations.rs
View File

@@ -4,7 +4,9 @@ use crate::types::OwnedValue;
use super::{
convert_dbtype_to_jsonb, convert_json_to_db_type, get_json_value, json_path_from_owned_value,
json_string_to_db_type, Val,
json_string_to_db_type,
jsonb::{DeleteOperation, InsertOperation, ReplaceOperation},
Conv, OutputVariant, Val,
};
/// Represents a single patch operation in the merge queue.
@@ -155,16 +157,17 @@ pub fn json_remove(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
return Ok(OwnedValue::Null);
}
let mut json = convert_dbtype_to_jsonb(&args[0], true)?;
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
for arg in &args[1..] {
if let Some(path) = json_path_from_owned_value(arg, true)? {
let _ = json.remove_by_path(&path);
let mut op = DeleteOperation::new();
let _ = json.operate_on_path(&path, &mut op);
}
}
let el_type = json.is_valid()?;
json_string_to_db_type(json, el_type, false, true)
json_string_to_db_type(json, el_type, OutputVariant::String)
}
pub fn jsonb_remove(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
@@ -172,10 +175,11 @@ pub fn jsonb_remove(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
return Ok(OwnedValue::Null);
}
let mut json = convert_dbtype_to_jsonb(&args[0], true)?;
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
for arg in &args[1..] {
if let Some(path) = json_path_from_owned_value(arg, true)? {
json.remove_by_path(&path)?;
let mut op = DeleteOperation::new();
let _ = json.operate_on_path(&path, &mut op);
}
}
@@ -187,20 +191,22 @@ pub fn json_replace(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
return Ok(OwnedValue::Null);
}
let mut json = convert_dbtype_to_jsonb(&args[0], true)?;
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
let other = args[1..].chunks_exact(2);
for chunk in other {
let path = json_path_from_owned_value(&chunk[0], true)?;
let value = convert_dbtype_to_jsonb(&chunk[1], false)?;
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
if let Some(path) = path {
let _ = json.replace_by_path(&path, value);
let mut op = ReplaceOperation::new(value);
let _ = json.operate_on_path(&path, &mut op);
}
}
let el_type = json.is_valid()?;
json_string_to_db_type(json, el_type, false, false)
json_string_to_db_type(json, el_type, super::OutputVariant::String)
}
pub fn jsonb_replace(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
@@ -208,19 +214,65 @@ pub fn jsonb_replace(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
return Ok(OwnedValue::Null);
}
let mut json = convert_dbtype_to_jsonb(&args[0], true)?;
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
let other = args[1..].chunks_exact(2);
for chunk in other {
let path = json_path_from_owned_value(&chunk[0], true)?;
let value = convert_dbtype_to_jsonb(&chunk[1], false)?;
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
if let Some(path) = path {
let _ = json.replace_by_path(&path, value);
let mut op = ReplaceOperation::new(value);
let _ = json.operate_on_path(&path, &mut op);
}
}
let el_type = json.is_valid()?;
json_string_to_db_type(json, el_type, false, true)
json_string_to_db_type(json, el_type, OutputVariant::Binary)
}
pub fn json_insert(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
if args.is_empty() {
return Ok(OwnedValue::Null);
}
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
let other = args[1..].chunks_exact(2);
for chunk in other {
let path = json_path_from_owned_value(&chunk[0], true)?;
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
if let Some(path) = path {
let mut op = InsertOperation::new(value);
let _ = json.operate_on_path(&path, &mut op);
}
}
let el_type = json.is_valid()?;
json_string_to_db_type(json, el_type, OutputVariant::String)
}
pub fn jsonb_insert(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
if args.is_empty() {
return Ok(OwnedValue::Null);
}
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
let other = args[1..].chunks_exact(2);
for chunk in other {
let path = json_path_from_owned_value(&chunk[0], true)?;
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
if let Some(path) = path {
let mut op = InsertOperation::new(value);
let _ = json.operate_on_path(&path, &mut op);
}
}
let el_type = json.is_valid()?;
json_string_to_db_type(json, el_type, OutputVariant::Binary)
}
#[cfg(test)]

1878
core/json/jsonb.rs
View File

File diff suppressed because it is too large Load Diff

569
core/json/mod.rs
View File

@@ -9,14 +9,13 @@ use crate::bail_constraint_error;
pub use crate::json::de::from_str;
use crate::json::error::Error as JsonError;
pub use crate::json::json_operations::{
json_patch, json_remove, json_replace, jsonb_remove, jsonb_replace,
json_insert, json_patch, json_remove, json_replace, jsonb_insert, jsonb_remove, jsonb_replace,
};
use crate::json::json_path::{json_path, JsonPath, PathElement};
pub use crate::json::ser::to_string;
use crate::types::{OwnedValue, Text, TextSubtype};
use crate::types::{OwnedValue, OwnedValueType, Text, TextSubtype};
use crate::{bail_parse_error, json::de::ordered_object};
use indexmap::IndexMap;
use jsonb::{ElementType, Jsonb, JsonbHeader};
use jsonb::{ElementType, Jsonb, JsonbHeader, PathOperationMode, SearchOperation, SetOperation};
use ser::to_string_pretty;
use serde::{Deserialize, Serialize};
use std::borrow::Cow;
@@ -37,6 +36,18 @@ pub enum Val {
Object(Vec<(String, Val)>),
}
enum Conv {
Strict,
NotStrict,
ToString,
}
enum OutputVariant {
ElementType,
Binary,
String,
}
pub fn get_json(json_value: &OwnedValue, indent: Option<&str>) -> crate::Result<OwnedValue> {
match json_value {
OwnedValue::Text(ref t) => {
@@ -76,7 +87,7 @@ pub fn get_json(json_value: &OwnedValue, indent: Option<&str>) -> crate::Result<
}
pub fn jsonb(json_value: &OwnedValue) -> crate::Result<OwnedValue> {
let jsonbin = convert_dbtype_to_jsonb(json_value, true);
let jsonbin = convert_dbtype_to_jsonb(json_value, Conv::Strict);
match jsonbin {
Ok(jsonbin) => Ok(OwnedValue::Blob(Rc::new(jsonbin.data()))),
Err(_) => {
@@ -85,24 +96,28 @@ pub fn jsonb(json_value: &OwnedValue) -> crate::Result<OwnedValue> {
}
}
fn convert_dbtype_to_jsonb(val: &OwnedValue, strict: bool) -> crate::Result<Jsonb> {
fn convert_dbtype_to_jsonb(val: &OwnedValue, strict: Conv) -> crate::Result<Jsonb> {
match val {
OwnedValue::Text(text) => {
let res = if text.subtype == TextSubtype::Json || strict {
let res = if text.subtype == TextSubtype::Json || matches!(strict, Conv::Strict) {
// Parse directly as JSON if it's already JSON subtype or strict mode is on
let json = if matches!(strict, Conv::ToString) {
let mut str = text.as_str().replace('"', "\\\"");
str.insert(0, '"');
str.push('"');
Jsonb::from_str(&str)
} else {
Jsonb::from_str(text.as_str())
};
json
} else {
// Handle as a string literal otherwise
let mut str = text.as_str().replace('"', "\\\"");
if &str == "null" {
// Special case for "null"
Jsonb::from_str(&str)
} else {
// Quote the string to make it a JSON string
str.insert(0, '"');
str.push('"');
Jsonb::from_str(&str)
}
};
res
}
@@ -144,108 +159,79 @@ fn get_json_value(json_value: &OwnedValue) -> crate::Result<Val> {
}
pub fn json_array(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
let mut s = String::new();
s.push('[');
let mut json = Jsonb::make_empty_array(values.len());
// TODO: use `convert_db_type_to_json` and map each value with that function,
// so we can construct a `Val::Array` with each value and then serialize it directly.
for (idx, value) in values.iter().enumerate() {
match value {
OwnedValue::Blob(_) => crate::bail_constraint_error!("JSON cannot hold BLOB values"),
OwnedValue::Text(t) => {
if t.subtype == TextSubtype::Json {
s.push_str(t.as_str());
} else {
match to_string(&t.as_str().to_string()) {
Ok(json) => s.push_str(&json),
Err(_) => crate::bail_parse_error!("malformed JSON"),
for value in values.iter() {
if matches!(value, OwnedValue::Blob(_)) {
crate::bail_constraint_error!("JSON cannot hold BLOB values")
}
let value = convert_dbtype_to_jsonb(value, Conv::NotStrict)?;
json.append_jsonb_to_end(value.data());
}
}
OwnedValue::Integer(i) => match to_string(&i) {
Ok(json) => s.push_str(&json),
Err(_) => crate::bail_parse_error!("malformed JSON"),
},
OwnedValue::Float(f) => match to_string(&f) {
Ok(json) => s.push_str(&json),
Err(_) => crate::bail_parse_error!("malformed JSON"),
},
OwnedValue::Null => s.push_str("null"),
_ => unreachable!(),
json.finalize_unsafe(ElementType::ARRAY)?;
json_string_to_db_type(json, ElementType::ARRAY, OutputVariant::ElementType)
}
if idx < values.len() - 1 {
s.push(',');
}
}
pub fn jsonb_array(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
let mut json = Jsonb::make_empty_array(values.len());
s.push(']');
Ok(OwnedValue::Text(Text::json(s)))
for value in values.iter() {
if matches!(value, OwnedValue::Blob(_)) {
crate::bail_constraint_error!("JSON cannot hold BLOB values")
}
let value = convert_dbtype_to_jsonb(value, Conv::NotStrict)?;
json.append_jsonb_to_end(value.data());
}
json.finalize_unsafe(ElementType::ARRAY)?;
json_string_to_db_type(json, ElementType::ARRAY, OutputVariant::Binary)
}
pub fn json_array_length(
json_value: &OwnedValue,
json_path: Option<&OwnedValue>,
) -> crate::Result<OwnedValue> {
let json = convert_dbtype_to_jsonb(json_value, true)?;
let mut json = convert_dbtype_to_jsonb(json_value, Conv::Strict)?;
if json_path.is_none() {
let result = json.array_len()?;
return Ok(OwnedValue::Integer(result as i64));
let len = json.array_len()?;
return Ok(OwnedValue::Integer(len as i64));
}
let path = json_path_from_owned_value(json_path.expect("We already checked none"), true)?;
if let Some(path) = path {
if let Ok(len) = json
.get_by_path(&path)
.and_then(|(json, _)| json.array_len())
{
let mut op = SearchOperation::new(json.len() / 2);
let _ = json.operate_on_path(&path, &mut op);
if let Ok(len) = op.result().array_len() {
return Ok(OwnedValue::Integer(len as i64));
}
}
Ok(OwnedValue::Null)
}
pub fn json_set(json: &OwnedValue, values: &[OwnedValue]) -> crate::Result<OwnedValue> {
let mut json_value = get_json_value(json)?;
pub fn json_set(args: &[OwnedValue]) -> crate::Result<OwnedValue> {
if args.is_empty() {
return Ok(OwnedValue::Null);
}
values
.chunks(2)
.map(|chunk| match chunk {
[path, value] => {
let path = json_path_from_owned_value(path, true)?;
let mut json = convert_dbtype_to_jsonb(&args[0], Conv::Strict)?;
let other = args[1..].chunks_exact(2);
for chunk in other {
let path = json_path_from_owned_value(&chunk[0], true)?;
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
let mut op = SetOperation::new(value);
if let Some(path) = path {
let new_value = match value {
OwnedValue::Text(
t @ Text {
subtype: TextSubtype::Text,
..
},
) => Val::String(t.as_str().to_string()),
_ => get_json_value(value)?,
};
let mut new_json_value = json_value.clone();
match create_and_mutate_json_by_path(&mut new_json_value, path, |val| match val
{
Target::Array(arr, index) => arr[index] = new_value.clone(),
Target::Value(val) => *val = new_value.clone(),
}) {
Some(_) => json_value = new_json_value,
_ => {}
let _ = json.operate_on_path(&path, &mut op);
}
}
Ok(())
}
_ => crate::bail_constraint_error!("json_set needs an odd number of arguments"),
})
.collect::<crate::Result<()>>()?;
let el_type = json.is_valid()?;
convert_json_to_db_type(&json_value, true)
json_string_to_db_type(json, el_type, OutputVariant::String)
}
/// Implements the -> operator. Always returns a proper JSON value.
@@ -256,10 +242,12 @@ pub fn json_arrow_extract(value: &OwnedValue, path: &OwnedValue) -> crate::Resul
}
if let Some(path) = json_path_from_owned_value(path, false)? {
let json = convert_dbtype_to_jsonb(value, true)?;
let extracted = json.get_by_path(&path);
if let Ok((json, _)) = extracted {
Ok(OwnedValue::Text(Text::json(json.to_string()?)))
let mut json = convert_dbtype_to_jsonb(value, Conv::Strict)?;
let mut op = SearchOperation::new(json.len());
let res = json.operate_on_path(&path, &mut op);
let extracted = op.result();
if res.is_ok() {
Ok(OwnedValue::Text(Text::json(extracted.to_string()?)))
} else {
Ok(OwnedValue::Null)
}
@@ -278,10 +266,21 @@ pub fn json_arrow_shift_extract(
return Ok(OwnedValue::Null);
}
if let Some(path) = json_path_from_owned_value(path, false)? {
let json = convert_dbtype_to_jsonb(value, true)?;
let extracted = json.get_by_path(&path);
if let Ok((json, element_type)) = extracted {
Ok(json_string_to_db_type(json, element_type, false, true)?)
let mut json = convert_dbtype_to_jsonb(value, Conv::Strict)?;
let mut op = SearchOperation::new(json.len());
let res = json.operate_on_path(&path, &mut op);
let extracted = op.result();
let element_type = match extracted.is_valid() {
Err(_) => return Ok(OwnedValue::Null),
Ok(el) => el,
};
if res.is_ok() {
Ok(json_string_to_db_type(
extracted,
element_type,
OutputVariant::ElementType,
)?)
} else {
Ok(OwnedValue::Null)
}
@@ -303,7 +302,7 @@ pub fn json_extract(value: &OwnedValue, paths: &[OwnedValue]) -> crate::Result<O
}
let (json, element_type) = jsonb_extract_internal(value, paths)?;
let result = json_string_to_db_type(json, element_type, false, true)?;
let result = json_string_to_db_type(json, element_type, OutputVariant::ElementType)?;
Ok(result)
}
@@ -318,7 +317,7 @@ pub fn jsonb_extract(value: &OwnedValue, paths: &[OwnedValue]) -> crate::Result<
}
let (json, element_type) = jsonb_extract_internal(value, paths)?;
let result = json_string_to_db_type(json, element_type, false, true)?;
let result = json_string_to_db_type(json, element_type, OutputVariant::ElementType)?;
Ok(result)
}
@@ -330,9 +329,17 @@ fn jsonb_extract_internal(
let null = Jsonb::from_raw_data(JsonbHeader::make_null().into_bytes().as_bytes());
if paths.len() == 1 {
if let Some(path) = json_path_from_owned_value(&paths[0], true)? {
let json = convert_dbtype_to_jsonb(value, true)?;
if let Ok((json, value_type)) = json.get_by_path(&path) {
return Ok((json, value_type));
let mut json = convert_dbtype_to_jsonb(value, Conv::Strict)?;
let mut op = SearchOperation::new(json.len());
let res = json.operate_on_path(&path, &mut op);
let extracted = op.result();
let element_type = match extracted.is_valid() {
Err(_) => return Ok((null, ElementType::NULL)),
Ok(el) => el,
};
if res.is_ok() {
return Ok((extracted, element_type));
} else {
return Ok((null, ElementType::NULL));
}
@@ -341,15 +348,18 @@ fn jsonb_extract_internal(
}
}
let json = convert_dbtype_to_jsonb(value, true)?;
let mut json = convert_dbtype_to_jsonb(value, Conv::Strict)?;
let mut result = Jsonb::make_empty_array(json.len());
// TODO: make an op to avoid creating new json for every path element
let paths = paths.iter().map(|p| json_path_from_owned_value(p, true));
for path in paths {
if let Some(path) = path? {
let fragment = json.get_by_path_raw(&path);
if let Ok(data) = fragment {
result.append_to_array_unsafe(data);
let mut op = SearchOperation::new(json.len());
let res = json.operate_on_path(&path, &mut op);
let extracted = op.result();
if res.is_ok() {
result.append_to_array_unsafe(&extracted.data());
} else {
result.append_to_array_unsafe(JsonbHeader::make_null().into_bytes().as_bytes());
}
@@ -357,24 +367,23 @@ fn jsonb_extract_internal(
return Ok((null, ElementType::NULL));
}
}
result.finalize_array_unsafe()?;
result.finalize_unsafe(ElementType::ARRAY)?;
Ok((result, ElementType::ARRAY))
}
fn json_string_to_db_type(
json: Jsonb,
element_type: ElementType,
raw_flag: bool,
raw_text: bool,
flag: OutputVariant,
) -> crate::Result<OwnedValue> {
let mut json_string = json.to_string()?;
if raw_flag {
if matches!(flag, OutputVariant::Binary) {
return Ok(OwnedValue::Blob(Rc::new(json.data())));
}
match element_type {
ElementType::ARRAY | ElementType::OBJECT => Ok(OwnedValue::Text(Text::json(json_string))),
ElementType::TEXT | ElementType::TEXT5 | ElementType::TEXTJ | ElementType::TEXTRAW => {
if raw_text {
if matches!(flag, OutputVariant::ElementType) {
json_string.remove(json_string.len() - 1);
json_string.remove(0);
Ok(OwnedValue::Text(Text {
@@ -446,43 +455,26 @@ fn convert_json_to_db_type(extracted: &Val, all_as_db: bool) -> crate::Result<Ow
}
}
/// Converts a DB value (`OwnedValue`) to a JSON representation (`Val`).
/// Note that when the internal text value is a json,
/// the returned `Val` will be an object. If the internal text value is a regular text,
/// then a string will be returned. This is useful to track if the value came from a json
/// function and therefore we must interpret it as json instead of raw text when working with it.
fn convert_db_type_to_json(value: &OwnedValue) -> crate::Result<Val> {
let val = match value {
OwnedValue::Null => Val::Null,
OwnedValue::Float(f) => Val::Float(*f),
OwnedValue::Integer(i) => Val::Integer(*i),
OwnedValue::Text(t) => match t.subtype {
// Convert only to json if the subtype is json (if we got it from another json function)
TextSubtype::Json => get_json_value(value)?,
TextSubtype::Text => Val::String(t.as_str().to_string()),
},
OwnedValue::Blob(_) => crate::bail_constraint_error!("JSON cannot hold BLOB values"),
unsupported_value => crate::bail_constraint_error!(
"JSON cannot hold this type of value: {unsupported_value:?}"
),
};
Ok(val)
}
pub fn json_type(value: &OwnedValue, path: Option<&OwnedValue>) -> crate::Result<OwnedValue> {
if let OwnedValue::Null = value {
return Ok(OwnedValue::Null);
}
if path.is_none() {
let json = convert_dbtype_to_jsonb(value, true)?;
let json = convert_dbtype_to_jsonb(value, Conv::Strict)?;
let element_type = json.is_valid()?;
return Ok(OwnedValue::Text(Text::json(element_type.into())));
}
if let Some(path) = json_path_from_owned_value(path.unwrap(), true)? {
let json = convert_dbtype_to_jsonb(value, true)?;
let mut json = convert_dbtype_to_jsonb(value, Conv::Strict)?;
if let Ok((_, element_type)) = json.get_by_path(&path) {
if let Ok(mut path) = json.navigate_path(&path, PathOperationMode::ReplaceExisting) {
let target = path.pop().expect("Should exist");
let element_type = if let Some(el_index) = target.get_array_index() {
json.element_type_at(el_index)
} else {
json.element_type_at(target.field_value_index)
}?;
Ok(OwnedValue::Text(Text::json(element_type.into())))
} else {
Ok(OwnedValue::Null)
@@ -533,98 +525,6 @@ fn json_path_from_owned_value(path: &OwnedValue, strict: bool) -> crate::Result<
Ok(Some(json_path))
}
enum Target<'a> {
Array(&'a mut Vec<Val>, usize),
Value(&'a mut Val),
}
fn create_and_mutate_json_by_path<F, R>(json: &mut Val, path: JsonPath, closure: F) -> Option<R>
where
F: FnOnce(Target) -> R,
{
find_or_create_target(json, &path).map(closure)
}
fn find_or_create_target<'a>(json: &'a mut Val, path: &JsonPath) -> Option<Target<'a>> {
let mut current = json;
for (i, key) in path.elements.iter().enumerate() {
let is_last = i == path.elements.len() - 1;
match key {
PathElement::Root() => continue,
PathElement::ArrayLocator(index) => match current {
Val::Array(arr) => {
if let Some(index) = match index {
Some(i) if *i < 0 => arr.len().checked_sub(i.unsigned_abs() as usize),
Some(i) => Some(*i as usize),
None => Some(arr.len()),
} {
if is_last {
if index == arr.len() {
arr.push(Val::Null);
}
if index >= arr.len() {
return None;
}
return Some(Target::Array(arr, index));
} else {
if index == arr.len() {
arr.push(
if matches!(path.elements[i + 1], PathElement::ArrayLocator(_))
{
Val::Array(vec![])
} else {
Val::Object(vec![])
},
);
}
if index >= arr.len() {
return None;
}
current = &mut arr[index];
}
} else {
return None;
}
}
_ => {
*current = Val::Array(vec![]);
}
},
PathElement::Key(key, _) => match current {
Val::Object(obj) => {
if let Some(pos) = &obj
.iter()
.position(|(k, v)| k == key && !matches!(v, Val::Removed))
{
let val = &mut obj[*pos].1;
current = val;
} else {
let element = if !is_last
&& matches!(path.elements[i + 1], PathElement::ArrayLocator(_))
{
Val::Array(vec![])
} else {
Val::Object(vec![])
};
obj.push((key.to_string(), element));
let index = obj.len() - 1;
current = &mut obj[index].1;
}
}
_ => {
return None;
}
},
}
}
Some(Target::Value(current))
}
pub fn json_error_position(json: &OwnedValue) -> crate::Result<OwnedValue> {
match json {
OwnedValue::Text(t) => match from_str::<Val>(t.as_str()) {
@@ -651,31 +551,52 @@ pub fn json_error_position(json: &OwnedValue) -> crate::Result<OwnedValue> {
/// The number of values must be even, and the first value of each pair (which represents the map key)
/// must be a TEXT value. The second value of each pair can be any JSON value (which represents the map value)
pub fn json_object(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
let value_map = values
.chunks(2)
.map(|chunk| match chunk {
[key, value] => {
let key = match key {
OwnedValue::Text(t) => t.as_str().to_string(),
_ => crate::bail_constraint_error!("labels must be TEXT"),
};
let json_val = convert_db_type_to_json(value)?;
Ok((key, json_val))
if values.len() % 2 != 0 {
bail_constraint_error!("json_object() requires an even number of arguments")
}
_ => crate::bail_constraint_error!("json_object requires an even number of values"),
})
.collect::<Result<IndexMap<String, Val>, _>>()?;
let mut json = Jsonb::make_empty_obj(values.len() * 50);
let result = crate::json::to_string(&value_map)?;
Ok(OwnedValue::Text(Text::json(result)))
for chunk in values.chunks_exact(2) {
if chunk[0].value_type() != OwnedValueType::Text {
bail_constraint_error!("json_object() labels must be TEXT")
}
let key = convert_dbtype_to_jsonb(&chunk[0], Conv::ToString)?;
json.append_jsonb_to_end(key.data());
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
json.append_jsonb_to_end(value.data());
}
json.finalize_unsafe(ElementType::OBJECT)?;
json_string_to_db_type(json, ElementType::OBJECT, OutputVariant::String)
}
pub fn jsonb_object(values: &[OwnedValue]) -> crate::Result<OwnedValue> {
if values.len() % 2 != 0 {
bail_constraint_error!("json_object() requires an even number of arguments")
}
let mut json = Jsonb::make_empty_obj(values.len() * 50);
for chunk in values.chunks_exact(2) {
if chunk[0].value_type() != OwnedValueType::Text {
bail_constraint_error!("json_object() labels must be TEXT")
}
let key = convert_dbtype_to_jsonb(&chunk[0], Conv::ToString)?;
json.append_jsonb_to_end(key.data());
let value = convert_dbtype_to_jsonb(&chunk[1], Conv::NotStrict)?;
json.append_jsonb_to_end(value.data());
}
json.finalize_unsafe(ElementType::OBJECT)?;
json_string_to_db_type(json, ElementType::OBJECT, OutputVariant::Binary)
}
pub fn is_json_valid(json_value: &OwnedValue) -> OwnedValue {
if matches!(json_value, OwnedValue::Null) {
return OwnedValue::Null;
}
convert_dbtype_to_jsonb(json_value, true)
convert_dbtype_to_jsonb(json_value, Conv::Strict)
.map(|_| OwnedValue::Integer(1))
.unwrap_or(OwnedValue::Integer(0))
}
@@ -1178,7 +1099,7 @@ mod tests {
let OwnedValue::Text(json_text) = result else {
panic!("Expected OwnedValue::Text");
};
assert_eq!(json_text.as_str(), r#"{"key":"value"}"#);
assert_eq!(json_text.as_str(), r#"{"key":"value","key":"value"}"#);
}
#[test]
@@ -1210,12 +1131,7 @@ mod tests {
let value = OwnedValue::build_text("value");
let input = vec![key.clone(), value, key];
match json_object(&input) {
Ok(_) => panic!("Expected error for odd number of values"),
Err(e) => assert!(e
.to_string()
.contains("json_object requires an even number of values")),
}
assert!(json_object(&input).is_err());
}
#[test]
@@ -1347,219 +1263,186 @@ mod tests {
#[test]
fn test_json_set_field_empty_object() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::build_text("$.field"),
OwnedValue::build_text("value"),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"field":"value"}"#)
);
assert_eq!(result.unwrap().to_text().unwrap(), r#"{"field":"value"}"#);
}
#[test]
fn test_json_set_replace_field() {
let result = json_set(
&OwnedValue::build_text(r#"{"field":"old_value"}"#),
&[
let result = json_set(&[
OwnedValue::build_text(r#"{"field":"old_value"}"#),
OwnedValue::build_text("$.field"),
OwnedValue::build_text("new_value"),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"field":"new_value"}"#)
result.unwrap().to_text().unwrap(),
r#"{"field":"new_value"}"#
);
}
#[test]
fn test_json_set_set_deeply_nested_key() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::build_text("$.object.doesnt.exist"),
OwnedValue::build_text("value"),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"object":{"doesnt":{"exist":"value"}}}"#)
result.unwrap().to_text().unwrap(),
r#"{"object":{"doesnt":{"exist":"value"}}}"#
);
}
#[test]
fn test_json_set_add_value_to_empty_array() {
let result = json_set(
&OwnedValue::build_text("[]"),
&[
let result = json_set(&[
OwnedValue::build_text("[]"),
OwnedValue::build_text("$[0]"),
OwnedValue::build_text("value"),
],
);
]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), OwnedValue::build_text(r#"["value"]"#));
assert_eq!(result.unwrap().to_text().unwrap(), r#"["value"]"#);
}
#[test]
fn test_json_set_add_value_to_nonexistent_array() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::build_text("$.some_array[0]"),
OwnedValue::Integer(123),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"some_array":[123]}"#)
result.unwrap().to_text().unwrap(),
r#"{"some_array":[123]}"#
);
}
#[test]
fn test_json_set_add_value_to_array() {
let result = json_set(
&OwnedValue::build_text("[123]"),
&[OwnedValue::build_text("$[1]"), OwnedValue::Integer(456)],
);
let result = json_set(&[
OwnedValue::build_text("[123]"),
OwnedValue::build_text("$[1]"),
OwnedValue::Integer(456),
]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), OwnedValue::build_text("[123,456]"));
assert_eq!(result.unwrap().to_text().unwrap(), "[123,456]");
}
#[test]
fn test_json_set_add_value_to_array_out_of_bounds() {
let result = json_set(
&OwnedValue::build_text("[123]"),
&[OwnedValue::build_text("$[200]"), OwnedValue::Integer(456)],
);
let result = json_set(&[
OwnedValue::build_text("[123]"),
OwnedValue::build_text("$[200]"),
OwnedValue::Integer(456),
]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), OwnedValue::build_text("[123]"));
assert_eq!(result.unwrap().to_text().unwrap(), "[123]");
}
#[test]
fn test_json_set_replace_value_in_array() {
let result = json_set(
&OwnedValue::build_text("[123]"),
&[OwnedValue::build_text("$[0]"), OwnedValue::Integer(456)],
);
let result = json_set(&[
OwnedValue::build_text("[123]"),
OwnedValue::build_text("$[0]"),
OwnedValue::Integer(456),
]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), OwnedValue::build_text("[456]"));
assert_eq!(result.unwrap().to_text().unwrap(), "[456]");
}
#[test]
fn test_json_set_null_path() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[OwnedValue::Null, OwnedValue::Integer(456)],
);
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::Null,
OwnedValue::Integer(456),
]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), OwnedValue::build_text("{}"));
assert_eq!(result.unwrap().to_text().unwrap(), "{}");
}
#[test]
fn test_json_set_multiple_keys() {
let result = json_set(
&OwnedValue::build_text("[123]"),
&[
let result = json_set(&[
OwnedValue::build_text("[123]"),
OwnedValue::build_text("$[0]"),
OwnedValue::Integer(456),
OwnedValue::build_text("$[1]"),
OwnedValue::Integer(789),
],
);
]);
assert!(result.is_ok());
assert_eq!(result.unwrap(), OwnedValue::build_text("[456,789]"));
}
#[test]
fn test_json_set_missing_value() {
let result = json_set(
&OwnedValue::build_text("[123]"),
&[OwnedValue::build_text("$[0]")],
);
assert!(result.is_err());
assert_eq!(result.unwrap().to_text().unwrap(), "[456,789]");
}
#[test]
fn test_json_set_add_array_in_nested_object() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::build_text("$.object[0].field"),
OwnedValue::Integer(123),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"object":[{"field":123}]}"#)
result.unwrap().to_text().unwrap(),
r#"{"object":[{"field":123}]}"#
);
}
#[test]
fn test_json_set_add_array_in_array_in_nested_object() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::build_text("$.object[0][0]"),
OwnedValue::Integer(123),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"object":[[123]]}"#)
);
assert_eq!(result.unwrap().to_text().unwrap(), r#"{"object":[[123]]}"#);
}
#[test]
fn test_json_set_add_array_in_array_in_nested_object_out_of_bounds() {
let result = json_set(
&OwnedValue::build_text("{}"),
&[
let result = json_set(&[
OwnedValue::build_text("{}"),
OwnedValue::build_text("$.object[123].another"),
OwnedValue::build_text("value"),
OwnedValue::build_text("$.field"),
OwnedValue::build_text("value"),
],
);
]);
assert!(result.is_ok());
assert_eq!(
result.unwrap(),
OwnedValue::build_text(r#"{"field":"value"}"#)
);
assert_eq!(result.unwrap().to_text().unwrap(), r#"{"field":"value"}"#,);
}
}

7
core/translate/expr.rs
View File

@@ -895,12 +895,15 @@ pub fn translate_expr(
)
}
JsonFunc::JsonArray
| JsonFunc::JsonbArray
| JsonFunc::JsonExtract
| JsonFunc::JsonSet
| JsonFunc::JsonbExtract
| JsonFunc::JsonReplace
| JsonFunc::JsonbReplace
| JsonFunc::JsonbRemove => translate_function(
| JsonFunc::JsonbRemove
| JsonFunc::JsonInsert
| JsonFunc::JsonbInsert => translate_function(
program,
args.as_deref().unwrap_or_default(),
referenced_tables,
@@ -950,7 +953,7 @@ pub fn translate_expr(
});
Ok(target_register)
}
JsonFunc::JsonObject => {
JsonFunc::JsonObject | JsonFunc::JsonbObject => {
let args = expect_arguments_even!(args, j);
translate_function(

44
core/vdbe/mod.rs
View File

@@ -48,15 +48,16 @@ use crate::util::{
use crate::vdbe::builder::CursorType;
use crate::vdbe::insn::Insn;
use crate::vector::{vector32, vector64, vector_distance_cos, vector_extract};
use crate::{bail_constraint_error, info, CheckpointStatus};
#[cfg(feature = "json")]
use crate::{
function::JsonFunc, json::get_json, json::is_json_valid, json::json_array,
json::json_array_length, json::json_arrow_extract, json::json_arrow_shift_extract,
json::json_error_position, json::json_extract, json::json_object, json::json_patch,
json::json_quote, json::json_remove, json::json_replace, json::json_set, json::json_type,
json::jsonb, json::jsonb_extract, json::jsonb_remove, json::jsonb_replace,
json::json_error_position, json::json_extract, json::json_insert, json::json_object,
json::json_patch, json::json_quote, json::json_remove, json::json_replace, json::json_set,
json::json_type, json::jsonb, json::jsonb_array, json::jsonb_extract, json::jsonb_insert,
json::jsonb_object, json::jsonb_remove, json::jsonb_replace,
};
use crate::{info, CheckpointStatus};
use crate::{
resolve_ext_path, Connection, MvCursor, MvStore, Result, TransactionState, DATABASE_VERSION,
};
@@ -2143,13 +2144,18 @@ impl Program {
Err(e) => return Err(e),
}
}
JsonFunc::JsonArray | JsonFunc::JsonObject => {
JsonFunc::JsonArray
| JsonFunc::JsonObject
| JsonFunc::JsonbArray
| JsonFunc::JsonbObject => {
let reg_values =
&state.registers[*start_reg..*start_reg + arg_count];
let json_func = match json_func {
JsonFunc::JsonArray => json_array,
JsonFunc::JsonObject => json_object,
JsonFunc::JsonbArray => jsonb_array,
JsonFunc::JsonbObject => jsonb_object,
_ => unreachable!(),
};
let json_result = json_func(reg_values);
@@ -2283,6 +2289,24 @@ impl Program {
state.registers[*dest] = OwnedValue::Null;
}
}
JsonFunc::JsonInsert => {
if let Ok(json) = json_insert(
&state.registers[*start_reg..*start_reg + arg_count],
) {
state.registers[*dest] = json;
} else {
state.registers[*dest] = OwnedValue::Null;
}
}
JsonFunc::JsonbInsert => {
if let Ok(json) = jsonb_insert(
&state.registers[*start_reg..*start_reg + arg_count],
) {
state.registers[*dest] = json;
} else {
state.registers[*dest] = OwnedValue::Null;
}
}
JsonFunc::JsonPretty => {
let json_value = &state.registers[*start_reg];
let indent = if arg_count > 1 {
@@ -2318,11 +2342,15 @@ impl Program {
state.registers[*dest] = json_str;
}
JsonFunc::JsonSet => {
if arg_count % 2 == 0 {
bail_constraint_error!(
"json_set() needs an odd number of arguments"
)
}
let reg_values =
&state.registers[*start_reg + 1..*start_reg + arg_count];
&state.registers[*start_reg..*start_reg + arg_count];
let json_result =
json_set(&state.registers[*start_reg], reg_values);
let json_result = json_set(reg_values);
match json_result {
Ok(json) => state.registers[*dest] = json,