Propagate info about hidden columns

testing/cli_tests/extensions.py

@@ -797,6 +797,137 @@ def test_tablestats():
     limbo.quit()
 
 
+def test_hidden_columns():
+    _test_hidden_columns(exec_name=None, ext_path="target/debug/libturso_ext_tests")
+    _test_hidden_columns(exec_name="sqlite3", ext_path="target/debug/liblimbo_sqlite_test_ext")
+
+
+def _test_hidden_columns(exec_name, ext_path):
+    console.info(f"Running test_hidden_columns for {ext_path}")
+
+    limbo = TestTursoShell(exec_name=exec_name,)
+    limbo.execute_dot(f".load {ext_path}")
+    limbo.execute_dot(
+        "create virtual table t using kv_store;",
+    )
+    limbo.run_test_fn(".schema", lambda res: "CREATE VIRTUAL TABLE t" in res)
+    limbo.run_test_fn(
+        "insert into t(key, value) values ('k0', 'v0');",
+        null,
+        "can insert if hidden column is not specified explicitly",
+    )
+    limbo.run_test_fn(
+        "insert into t(key, value) values ('k1', 'v1');",
+        null,
+        "can insert if hidden column is not specified explicitly",
+    )
+    limbo.run_test_fn(
+        "select comment from t where key = 'k0';",
+        lambda res: "auto-generated" == res,
+        "can select a hidden column from kv_store",
+    )
+    limbo.run_test_fn(
+        "select comment from (select * from t where key = 'k0');",
+        lambda res: "Column comment not found" in res or "no such column: comment" in res,
+        "hidden columns are not exposed by subqueries by default",
+    )
+    limbo.run_test_fn(
+        "select * from (select comment from t where key = 'k0');",
+        lambda res: "auto-generated" == res,
+        "can select hidden column exposed by subquery",
+    )
+    limbo.run_test_fn(
+        "insert into t(comment, key, value) values ('my comment', 'hidden', 'test');",
+        null,
+        "can insert if a hidden column is specified explicitly",
+    )
+    limbo.run_test_fn(
+        "select comment from t where key = 'hidden';",
+        lambda res: "my comment" == res,
+        "can select a hidden column from kv_store",
+    )
+    limbo.run_test_fn(
+        "select * from t where key = 'hidden';",
+        lambda res: "hidden|test" == res,
+        "hidden column is excluded from * expansion",
+    )
+    limbo.run_test_fn(
+        "select t.* from t where key = 'hidden';",
+        lambda res: "hidden|test" == res,
+        "hidden column is excluded from <table name>.* expansion",
+    )
+    limbo.run_test_fn(
+        "insert into t(comment, key, value) values ('insert_hidden', 'test');",
+        lambda res: "2 values for 3 columns" in res,
+        "fails when number of values does not match number of specified columns",
+    )
+    limbo.run_test_fn(
+        "update t set comment = 'updated comment' where key = 'hidden';",
+        null,
+        "can update a hidden column if specified explicitly",
+    )
+    limbo.run_test_fn(
+        "select comment from t where key = 'hidden';",
+        lambda res: "updated comment" == res,
+    )
+    limbo.run_test_fn(
+        "PRAGMA table_info=t;",
+        lambda res: "0|key|TEXT|0|TURSO|1\n1|value|TEXT|0|TURSO|0" == res,
+        "hidden columns are not listed in the dataset returned by 'PRAGMA table_info'",
+    )
+    limbo.run_test_fn(
+        "select comment, count(*) from t group by comment;",
+        lambda res: "auto-generated|2\nupdated comment|1" == res,
+        "can use hidden columns in aggregations",
+    )
+
+    # ORDER BY
+    limbo.execute_dot("CREATE VIRTUAL TABLE o USING kv_store;")
+    limbo.run_test_fn(".schema", lambda res: "CREATE VIRTUAL TABLE o" in res)
+    limbo.execute_dot("INSERT INTO o(comment, key, value) VALUES ('0', '5', 'a');")
+    limbo.execute_dot("INSERT INTO o(comment, key, value) VALUES ('1', '4', 'b');")
+    limbo.execute_dot("INSERT INTO o(comment, key, value) VALUES ('2', '3', 'c');")
+    limbo.run_test_fn(
+        "SELECT * FROM o ORDER BY comment;",
+        lambda res: "5|a\n4|b\n3|c" == res,
+    )
+    limbo.run_test_fn(
+        "SELECT * FROM o ORDER BY 0;",
+        lambda res: "invalid column index: 0" in res or "term out of range - should be between 1 and 2" in res,
+    )
+    limbo.run_test_fn(
+        "SELECT * FROM o ORDER BY 1;",
+        lambda res: "3|c\n4|b\n5|a" == res,
+    )
+
+    # JOINs
+    limbo.execute_dot("CREATE TABLE r (comment, key, value);")
+    limbo.execute_dot("INSERT INTO r VALUES ('comment0', '2', '3');")
+    limbo.execute_dot("INSERT INTO r VALUES ('comment1', '4', '5');")
+    limbo.execute_dot("CREATE VIRTUAL TABLE l USING kv_store;")
+    limbo.run_test_fn(".schema", lambda res: "CREATE VIRTUAL TABLE l" in res)
+    limbo.execute_dot("INSERT INTO l(comment, key, value) values ('comment1', '2', '3');")
+    limbo.run_test_fn(
+        "SELECT * FROM l NATURAL JOIN r;",
+        lambda res: "2|3|comment0" == res,
+    )
+    limbo.run_test_fn(
+        "SELECT * FROM l JOIN r USING (comment);",
+        lambda res: "2|3|4|5" == res,
+    )
+    limbo.run_test_fn(
+        "SELECT * FROM l JOIN r ON l.comment = r.comment;",
+        lambda res: "2|3|comment1|4|5" == res,
+    )
+    # TODO: Limbo panics for:
+    # - SELECT * FROM l NATURAL JOIN r NATURAL JOIN r;
+    # - SELECT * FROM l NATURAL JOIN r NATURAL JOIN l;
+    # - SELECT * FROM r NATURAL JOIN l;
+    # - SELECT * FROM r NATURAL JOIN l NATURAL JOIN r;
+
+    limbo.quit()
+
+
 def main():
     try:
         test_regexp()
@@ -812,6 +943,7 @@ def main():
         test_create_virtual_table()
         test_csv()
         test_tablestats()
+        test_hidden_columns()
     except Exception as e:
         console.error(f"Test FAILED: {e}")
         cleanup()