Merge 'Implement strftime function' from Pedro Muniz

This PR implements the strftime scalar function. It uses underneath the hood chrono's strftime function and the already existent modifier logic. The caveat of using chrono's strftime implementation is that it supports some additional syntax that is not supported in sqlite, such as precision modifiers (%.3f). If this is a deal breaker for this function to be implemented at the moment, I will then write a strftime implementation from scratch. Closes #778
2025-12-29 22:14:23 +01:00 · 2025-01-26 08:47:44 +02:00
parent ef466cbdf7 9d858f5cb3
commit 538b3ef823
6 changed files with 243 additions and 19 deletions
--- a/COMPAT.md
+++ b/COMPAT.md
@@ -4,16 +4,24 @@ This document describes the compatibility of Limbo with SQLite.

 ## Table of contents:

- [Features](#features)
- [SQLite query language](#sqlite-query-language)
+- [Compatibility with SQLite](#compatibility-with-sqlite)
+  - [Table of contents:](#table-of-contents)
+  - [Features](#features)
+  - [SQLite query language](#sqlite-query-language)
    - [Statements](#statements)
-      - [PRAGMA Statements](#pragma)
+      - [PRAGMA](#pragma)
    - [Expressions](#expressions)
-    - [Functions](#functions)
- [SQLite C API](#sqlite-c-api)
- [SQLite VDBE opcodes](#sqlite-vdbe-opcodes)
- [Extensions](#extensions)
+    - [SQL functions](#sql-functions)
+      - [Scalar functions](#scalar-functions)
+      - [Mathematical functions](#mathematical-functions)
+      - [Aggregate functions](#aggregate-functions)
+      - [Date and time functions](#date-and-time-functions)
+      - [JSON functions](#json-functions)
+  - [SQLite C API](#sqlite-c-api)
+  - [SQLite VDBE opcodes](#sqlite-vdbe-opcodes)
+  - [Extensions](#extensions)
    - [UUID](#uuid)
+    - [regexp](#regexp)

 ## Features

@@ -308,7 +316,7 @@ Feature support of [sqlite expr syntax](https://www.sqlite.org/lang_expr.html).
 | datetime()  | Yes     | partially supports modifiers |
 | julianday() | Partial | does not support modifiers   |
 | unixepoch() | Partial | does not support modifiers   |
-| strftime()  | No      |                              |
+| strftime()  | Yes     | partially supports modifiers |
 | timediff()  | No      |                              |

 Modifiers:
--- a/core/function.rs
+++ b/core/function.rs
@@ -213,6 +213,7 @@ pub enum ScalarFunc {
    Replace,
    #[cfg(not(target_family = "wasm"))]
    LoadExtension,
+    StrfTime,
 }

 impl Display for ScalarFunc {
@@ -264,6 +265,7 @@ impl Display for ScalarFunc {
            Self::DateTime => "datetime".to_string(),
            #[cfg(not(target_family = "wasm"))]
            Self::LoadExtension => "load_extension".to_string(),
+            Self::StrfTime => "strftime".to_string(),
        };
        write!(f, "{}", str)
    }
@@ -554,6 +556,7 @@ impl Func {
            "trunc" => Ok(Self::Math(MathFunc::Trunc)),
            #[cfg(not(target_family = "wasm"))]
            "load_extension" => Ok(Self::Scalar(ScalarFunc::LoadExtension)),
+            "strftime" => Ok(Self::Scalar(ScalarFunc::StrfTime)),
            _ => crate::bail_parse_error!("no such function: {}", name),
        }
    }
--- a/core/translate/expr.rs
+++ b/core/translate/expr.rs
@@ -1531,6 +1531,26 @@ pub fn translate_expr(
                            });
                            Ok(target_register)
                        }
+                        ScalarFunc::StrfTime => {
+                            if let Some(args) = args {
+                                for arg in args.iter() {
+                                    // register containing result of each argument expression
+                                    let _ = translate_and_mark(
+                                        program,
+                                        referenced_tables,
+                                        arg,
+                                        resolver,
+                                    )?;
+                                }
+                            }
+                            program.emit_insn(Insn::Function {
+                                constant_mask: 0,
+                                start_reg: target_register + 1,
+                                dest: target_register,
+                                func: func_ctx,
+                            });
+                            Ok(target_register)
+                        }
                    }
                }
                Func::Math(math_func) => match math_func.arity() {
--- a/core/vdbe/datetime.rs
+++ b/core/vdbe/datetime.rs
@@ -22,24 +22,44 @@ pub fn exec_datetime_full(values: &[OwnedValue]) -> OwnedValue {
    exec_datetime(values, DateTimeOutput::DateTime)
 }

+#[inline(always)]
+pub fn exec_strftime(values: &[OwnedValue]) -> OwnedValue {
+    if values.is_empty() {
+        return OwnedValue::Null;
+    }
+
+    let format_str = match &values[0] {
+        OwnedValue::Text(text) => text.value.to_string(),
+        OwnedValue::Integer(num) => num.to_string(),
+        OwnedValue::Float(num) => format!("{:.14}", num),
+        _ => return OwnedValue::Null,
+    };
+
+    exec_datetime(&values[1..], DateTimeOutput::StrfTime(format_str))
+}
+
 enum DateTimeOutput {
    Date,
    Time,
    DateTime,
+    // Holds the format string
+    StrfTime(String),
 }

 fn exec_datetime(values: &[OwnedValue], output_type: DateTimeOutput) -> OwnedValue {
    if values.is_empty() {
-        return OwnedValue::build_text(Rc::new(
-            parse_naive_date_time(&OwnedValue::build_text(Rc::new("now".to_string())))
-                .unwrap()
-                .format(match output_type {
-                    DateTimeOutput::DateTime => "%Y-%m-%d %H:%M:%S",
-                    DateTimeOutput::Time => "%H:%M:%S",
-                    DateTimeOutput::Date => "%Y-%m-%d",
-                })
-                .to_string(),
-        ));
+        let now =
+            parse_naive_date_time(&OwnedValue::build_text(Rc::new("now".to_string()))).unwrap();
+
+        let formatted_str = match output_type {
+            DateTimeOutput::DateTime => now.format("%Y-%m-%d %H:%M:%S").to_string(),
+            DateTimeOutput::Time => now.format("%H:%M:%S").to_string(),
+            DateTimeOutput::Date => now.format("%Y-%m-%d").to_string(),
+            DateTimeOutput::StrfTime(ref format_str) => strftime_format(&now, format_str),
+        };
+
+        // Parse here
+        return OwnedValue::build_text(Rc::new(formatted_str));
    }
    if let Some(mut dt) = parse_naive_date_time(&values[0]) {
        // if successful, treat subsequent entries as modifiers
@@ -95,6 +115,31 @@ fn format_dt(dt: NaiveDateTime, output_type: DateTimeOutput, subsec: bool) -> St
                dt.format("%Y-%m-%d %H:%M:%S").to_string()
            }
        }
+        DateTimeOutput::StrfTime(format_str) => strftime_format(&dt, &format_str),
+    }
+}
+
+// Not as fast as if the formatting was native to chrono, but a good enough
+// for now, just to have the feature implemented
+fn strftime_format(dt: &NaiveDateTime, format_str: &str) -> String {
+    use std::fmt::Write;
+    // Necessary to remove %f and %J that are exclusive formatters to sqlite
+    // Chrono does not support them, so it is necessary to replace the modifiers manually
+
+    // Sqlite uses 9 decimal places for julianday in strftime
+    let copy_format = format_str
+        .to_string()
+        .replace("%J", &format!("{:.9}", to_julian_day_exact(dt)));
+    // Just change the formatting here to have fractional seconds using chrono builtin modifier
+    let copy_format = copy_format.replace("%f", "%S.%3f");
+
+    // The write! macro is used here as chrono's format can panic if the formatting string contains
+    // unknown specifiers. By using a writer, we can catch the panic and handle the error
+    let mut formatted = String::new();
+    match write!(formatted, "{}", dt.format(&copy_format)) {
+        Ok(_) => formatted,
+        // On sqlite when the formatting fails nothing is printed
+        Err(_) => "".to_string(),
    }
 }

@@ -1729,4 +1774,7 @@ mod tests {
            .naive_utc();
        assert!(is_leap_second(&dt));
    }
+
+    #[test]
+    fn test_strftime() {}
 }
--- a/core/vdbe/mod.rs
+++ b/core/vdbe/mod.rs
@@ -44,7 +44,9 @@ use crate::{
    json::json_extract, json::json_object, json::json_type,
 };
 use crate::{resolve_ext_path, Connection, Result, Rows, TransactionState, DATABASE_VERSION};
-use datetime::{exec_date, exec_datetime_full, exec_julianday, exec_time, exec_unixepoch};
+use datetime::{
+    exec_date, exec_datetime_full, exec_julianday, exec_strftime, exec_time, exec_unixepoch,
+};
 use insn::{
    exec_add, exec_and, exec_bit_and, exec_bit_not, exec_bit_or, exec_boolean_not, exec_concat,
    exec_divide, exec_multiply, exec_or, exec_remainder, exec_shift_left, exec_shift_right,
@@ -2007,6 +2009,12 @@ impl Program {
                                    conn.load_extension(ext)?;
                                }
                            }
+                            ScalarFunc::StrfTime => {
+                                let result = exec_strftime(
+                                    &state.registers[*start_reg..*start_reg + arg_count],
+                                );
+                                state.registers[*dest] = result;
+                            }
                        },
                        crate::function::Func::External(f) => match f.func {
                            ExtFunc::Scalar(f) => {
--- a/testing/scalar-functions-datetime.test
+++ b/testing/scalar-functions-datetime.test
@@ -443,3 +443,140 @@ do_execsql_test julianday-time-only {
 #    SELECT julianday('2023-05-18');
 #} {2460082.5}

+
+
+
+# Strftime tests
+
+
+set FMT {%d,%e,%f,%F,%G,%g,%H,%I,%j,%J,%k,%l,%i,%m,%M,%p,%P,%R,%s,%S,%T,%U,%u,%V,%w,%W,%Y,%%}
+
+do_execsql_test strftime-day {
+    SELECT strftime('%d', '2025-01-23T13:10:30.567');
+} {23}
+
+do_execsql_test strftime-day-without-leading-zero-1 {
+    SELECT strftime('%e', '2025-01-23T13:10:30.567');
+} {23}
+
+do_execsql_test strftime-day-without-leading-zero-2 {
+    SELECT strftime('%e', '2025-01-02T13:10:30.567');
+} {" 2"} 
+# TODO not a typo in sqlite there is also a space 
+
+do_execsql_test strftime-fractional-seconds {
+    SELECT strftime('%f', '2025-01-02T13:10:30.567');
+} {30.567}
+
+do_execsql_test strftime-iso-8601-date {
+    SELECT strftime('%F', '2025-01-23T13:10:30.567');
+} {2025-01-23}
+
+do_execsql_test strftime-iso-8601-year {
+    SELECT strftime('%G', '2025-01-23T13:10:30.567');
+} {2025}
+
+do_execsql_test strftime-iso-8601-year-2_digit {
+    SELECT strftime('%g', '2025-01-23T13:10:30.567');
+} {25}
+
+do_execsql_test strftime-hour {
+    SELECT strftime('%H', '2025-01-23T13:10:30.567');
+} {13}
+
+do_execsql_test strftime-hour-12-hour-clock {
+    SELECT strftime('%I', '2025-01-23T13:10:30.567');
+} {01}
+
+do_execsql_test strftime-day-of-year {
+    SELECT strftime('%j', '2025-01-23T13:10:30.567');
+} {023}
+
+do_execsql_test strftime-julianday {
+    SELECT strftime('%J', '2025-01-23T13:10:30.567');
+} {2460699.048964896}
+
+do_execsql_test strftime-hour-without-leading-zero-1 {
+    SELECT strftime('%k', '2025-01-23T13:10:30.567');
+} {13}
+
+do_execsql_test strftime-hour-without-leading-zero-2 {
+    SELECT strftime('%k', '2025-01-23T02:10:30.567');
+} {" 2"}
+
+do_execsql_test strftime-hour-12-hour-clock-without-leading-zero-2 {
+    SELECT strftime('%l', '2025-01-23T13:10:30.567');
+} {" 1"}
+
+do_execsql_test strftime-month {
+    SELECT strftime('%m', '2025-01-23T13:10:30.567');
+} {01}
+
+do_execsql_test strftime-minute {
+    SELECT strftime('%M', '2025-01-23T13:14:30.567');
+} {14}
+
+do_execsql_test strftime-am-pm=1 {
+    SELECT strftime('%p', '2025-01-23T11:14:30.567');
+} {AM}
+
+do_execsql_test strftime-am-pm-2 {
+    SELECT strftime('%p', '2025-01-23T13:14:30.567');
+} {PM}
+
+do_execsql_test strftime-am-pm-lower-1 {
+    SELECT strftime('%P', '2025-01-23T11:14:30.567');
+} {am}
+
+do_execsql_test strftime-am-pm-lower-2 {
+    SELECT strftime('%P', '2025-01-23T13:14:30.567');
+} {pm}
+
+do_execsql_test strftime-iso8601-time {
+    SELECT strftime('%R', '2025-01-23T13:14:30.567');
+} {13:14}
+
+do_execsql_test strftime-seconds-since-epoch {
+    SELECT strftime('%s', '2025-01-23T13:14:30.567');
+} {1737638070}
+
+do_execsql_test strftime-seconds {
+    SELECT strftime('%S', '2025-01-23T13:14:30.567');
+} {30}
+
+do_execsql_test strftime-iso8601-with-seconds {
+    SELECT strftime('%T', '2025-01-23T13:14:30.567');
+} {13:14:30}
+
+do_execsql_test strftime-week-year-start-sunday {
+    SELECT strftime('%U', '2025-01-23T13:14:30.567');
+} {03}
+
+do_execsql_test strftime-day-week-start-monday {
+    SELECT strftime('%u', '2025-01-23T13:14:30.567');
+} {4}
+
+do_execsql_test strftime-iso8601-week-year {
+    SELECT strftime('%V', '2025-01-23T13:14:30.567');
+} {04}
+
+do_execsql_test strftime-day-week-start-sunday {
+    SELECT strftime('%w', '2025-01-23T13:14:30.567');
+} {4}
+
+do_execsql_test strftime-day-week-start-sunday {
+    SELECT strftime('%w', '2025-01-23T13:14:30.567');
+} {4}
+
+do_execsql_test strftime-week-year-start-sunday {
+    SELECT strftime('%W', '2025-01-23T13:14:30.567');
+} {03}
+
+do_execsql_test strftime-year {
+    SELECT strftime('%Y', '2025-01-23T13:14:30.567');
+} {2025}
+
+do_execsql_test strftime-percent {
+    SELECT strftime('%%', '2025-01-23T13:14:30.567');
+} {%}
+