improve handling of double quotes

I ended up hitting #1974 today and wanted to fix it. I worked with
Claude to generate a more comprehensive set of queries that could fail
aside from just the insert query described in the issue. He got most of
them right - lots of cases were indeed failing. The ones that were
gibberish, he told me I was absolutely right for pointing out they were
bad.

But alas. With the test cases generated, we can work on fixing it. The
place where the assertion was hit, all we need to do there is return
true (but we assert that this is indeed a string literal, it shouldn't
be anything else at this point).

There are then just a couple of places where we need to make sure we
handle double quotes correctly. We already tested for single quotes in a
couple of places, but never for double quotes.

There is one funny corner case where you can just select "col" from tbl,
and if there is no column "col" on the table, that is treated as a
string literal. We handle that too.

Fixes #1974

testing/insert.test

@@ -409,4 +409,52 @@ if {[info exists ::env(SQLITE_EXEC)] && ($::env(SQLITE_EXEC) eq "scripts/limbo-s
         INSERT INTO t VALUES (replace(hex(zeroblob(1000)), '00', 'a') || 'h', 8);
         SELECT COUNT(*) FROM t WHERE x >= replace(hex(zeroblob(100)), '00', 'a');
     } {8}
-}
+}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} double-quote-string-literals {
+    .dbconfig dqs_dml on
+    CREATE TABLE test (id INTEGER, name TEXT);
+    INSERT INTO test (id, name) VALUES (1, "Dave");
+    INSERT INTO test (id,name) VALUES (2, 'Alice');
+    SELECT * FROM test ORDER BY id;
+} {1|Dave
+2|Alice}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} mixed-quote-types {
+    .dbconfig dqs_dml on
+    CREATE TABLE mixed (a TEXT, b TEXT, c TEXT);
+    INSERT INTO mixed (a,b,c) VALUES ("double", 'single', "another");
+    SELECT * FROM mixed;
+} {double|single|another}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} double-quote-regression-original-case {
+    .dbconfig dqs_dml on
+    CREATE TABLE users (
+        id INTEGER,
+        name TEXT,
+        email TEXT
+    );
+    INSERT INTO users (name) values ("Dave");
+    SELECT name FROM users;
+} {Dave}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} double-quote-multiple-rows {
+    .dbconfig dqs_dml on
+    CREATE TABLE items (id INTEGER, description TEXT, category TEXT);
+    INSERT INTO items (id, description, category) VALUES
+        (1, "First_item", "category_a"),
+        (2, 'Second_item', "category_b"),
+        (3, "Third_item", 'category_c');
+    SELECT id, description, category FROM items ORDER BY id;
+} {1|First_item|category_a
+2|Second_item|category_b
+3|Third_item|category_c}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} double-quote-inner-quotes-preservation {
+    .dbconfig dqs_dml on
+    CREATE TABLE inner_quotes_test (id INTEGER, content TEXT);
+    INSERT INTO inner_quotes_test VALUES (1, '"foo"');
+    INSERT INTO inner_quotes_test VALUES (2, "'bar'");
+    SELECT id, content FROM inner_quotes_test ORDER BY id;
+} {1|"foo"
+2|'bar'}

testing/select.test

@@ -617,3 +617,20 @@ do_execsql_test_on_specific_db {:memory:} select-no-match-in-leaf-page {
 2
 2}
 
+# Regression tests for double-quoted strings in SELECT statements
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} select-double-quotes-values {
+    .dbconfig dqs_dml on
+    SELECT * FROM (VALUES ("select", "test"), ("double", "quotes"));
+} {select|test
+double|quotes}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} select-double-quotes-no-column {
+    .dbconfig dqs_dml on
+    SELECT "first"
+} {first}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} select-double-quotes-literal {
+    .dbconfig dqs_dml on
+    SELECT "literal_string" AS col;
+} {literal_string}
+

testing/tester.tcl

@@ -11,8 +11,8 @@ proc test_put {msg db test_name} {
 }
 
 proc evaluate_sql {sqlite_exec db_name sql} {
-    set command [list $sqlite_exec $db_name $sql]
-    set output [exec {*}$command]
+    set command [list $sqlite_exec $db_name]
+    set output [exec echo $sql | {*}$command]
     return $output
 }
 
@@ -69,6 +69,26 @@ proc do_execsql_test_on_specific_db {db_name test_name sql_statements expected_o
     run_test $::sqlite_exec $db_name $combined_sql $combined_expected_output
 }
 
+proc run_test_skip_lines {sqlite_exec skip_lines db_name sql expected_output} {
+    set actual_output [evaluate_sql $sqlite_exec $db_name $sql]
+    set lines [split $actual_output "\n"]
+    set actual_without_skipped [join [lrange $lines $skip_lines end] "\n"]
+    if {$actual_without_skipped ne $expected_output} {
+        error_put $sql
+        puts "returned '$actual_without_skipped'"
+        puts "expected '$expected_output'"
+        exit 1
+    }
+}
+
+proc do_execsql_test_skip_lines_on_specific_db {skip_lines db_name test_name sql_statements expected_outputs} {
+    test_put "Running test" $db_name $test_name
+    set combined_sql [string trim $sql_statements]
+    set combined_expected_output [join $expected_outputs "\n"]
+    run_test_skip_lines $::sqlite_exec $skip_lines $db_name $combined_sql $combined_expected_output
+}
+
+
 proc within_tolerance {actual expected tolerance} {
     expr {abs($actual - $expected) <= $tolerance}
 }

testing/values.test

@@ -25,4 +25,26 @@ do_execsql_test values-in-from {
 do_execsql_test values-in-join {
   select * from (values(1, 2)) join (values(3, 4), (5, 6));
 } {1|2|3|4
-  1|2|5|6};
+  1|2|5|6};
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} values-double-quotes-simple {
+    .dbconfig dqs_dml on
+    VALUES ("double_quoted_string");
+} {double_quoted_string}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} values-double-quotes-multiple {
+    .dbconfig dqs_dml on
+    VALUES ("first", "second"), ("third", "fourth");
+} {first|second
+third|fourth}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} values-mixed-quotes {
+    .dbconfig dqs_dml on
+    VALUES ("double", 'single'), ('single', "double");
+} {double|single
+single|double}
+
+do_execsql_test_skip_lines_on_specific_db 1 {:memory:} values-double-quotes-subquery {
+    .dbconfig dqs_dml on
+    SELECT * FROM (VALUES ("subquery_string"));
+} {subquery_string}