datastore: change keys into an array.

We store this internally as hex, since shelve insists on string keys.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>

datastore/datastore-plugin.py

@@ -6,6 +6,7 @@ from pyln.client import Plugin, RpcException
 from collections import namedtuple
 import os
 import shelve
+from typing import Optional, Sequence, List, Union
 
 # Error codes
 DATASTORE_DEL_DOES_NOT_EXIST = 1200
@@ -13,16 +14,41 @@ DATASTORE_DEL_WRONG_GENERATION = 1201
 DATASTORE_UPDATE_ALREADY_EXISTS = 1202
 DATASTORE_UPDATE_DOES_NOT_EXIST = 1203
 DATASTORE_UPDATE_WRONG_GENERATION = 1204
+DATASTORE_UPDATE_HAS_CHILDREN = 1205
+DATASTORE_UPDATE_NO_CHILDREN = 1206
 
 plugin = Plugin()
 Entry = namedtuple('Entry', ['generation', 'data'])
 
-def datastore_entry(key, entry: Entry):
+
+# A singleton to most commands turns into a [].
+def normalize_key(key: Union[Sequence[str], str]) -> List[str]:
+    if not isinstance(key, list) and not isinstance(key, tuple):
+        key = [key]
+    return key
+
+# We turn list into nul-separated hexbytes for storage
+def key_to_hex(key: Sequence[str]) -> str:
+    return b'\0'.join([bytes(k, encoding='utf8') for k in key]).hex()
+
+
+def hex_to_key(hexstr: str) -> List[str]:
+    return [b.decode() for b in bytes.fromhex(hexstr).split(b'\0')]
+
+
+def datastore_entry(key: Sequence[str], entry: Optional[Entry]):
     """Return a dict representing the entry"""
 
+    if not isinstance(key, list) and not isinstance(key, tuple):
+        key = [key]
+
+    ret = {'key': key}
+
+    if entry is not None:
         # Entry may be a simple tuple; convert
         entry = Entry(*entry)
-    ret = {'key': key, 'generation': entry.generation, 'hex': entry.data.hex()}
+        ret['generation'] = entry.generation
+        ret['hex'] = entry.data.hex()
 
         # FFS, Python3 seems happy with \0 in UTF-8.
         if 0 not in entry.data:
@@ -38,6 +64,8 @@ def datastore(plugin, key, string=None, hex=None, mode="must-create", generation
     """Add/modify a {key} and {hex}/{string} data to the data store,
 optionally insisting it be {generation}"""
 
+    key = normalize_key(key)
+    khex = key_to_hex(key)
     if string is not None:
         if hex is not None:
             raise RpcException("Cannot specify both string or hex")
@@ -48,11 +76,13 @@ optionally insisting it be {generation}"""
         data = bytes.fromhex(hex)
 
     if mode == "must-create":
-        if key in plugin.datastore:
+        if khex in plugin.datastore:
-            raise RpcException("already exists", DATASTORE_UPDATE_ALREADY_EXISTS)
+            raise RpcException("already exists",
+                               DATASTORE_UPDATE_ALREADY_EXISTS)
     elif mode == "must-replace":
-        if key not in plugin.datastore:
+        if khex not in plugin.datastore:
-            raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST)
+            raise RpcException("does not exist",
+                               DATASTORE_UPDATE_DOES_NOT_EXIST)
     elif mode == "create-or-replace":
         if generation is not None:
             raise RpcException("generation only valid with"
@@ -62,57 +92,88 @@ optionally insisting it be {generation}"""
         if generation is not None:
             raise RpcException("generation only valid with"
                                " must-create/must-replace")
-        if key not in plugin.datastore:
+        if khex not in plugin.datastore:
-            raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST)
+            raise RpcException("does not exist",
-        data = plugin.datastore[key].data + data
+                               DATASTORE_UPDATE_DOES_NOT_EXIST)
+        data = plugin.datastore[khex].data + data
     elif mode == "create-or-append":
         if generation is not None:
             raise RpcException("generation only valid with"
                                " must-create/must-replace")
-        data = plugin.datastore.get(key, Entry(0, bytes())).data + data
+        data = plugin.datastore.get(khex, Entry(0, bytes())).data + data
     else:
         raise RpcException("invalid mode")
 
-    if key in plugin.datastore:
+    # Make sure parent doesn't exist
-        entry = plugin.datastore[key]
+    parent = [key[0]]
+    for i in range(1, len(key)):
+        if key_to_hex(parent) in plugin.datastore:
+            raise RpcException("Parent key [{}] exists".format(','.join(parent)),
+                               DATASTORE_UPDATE_NO_CHILDREN)
+        parent += [key[i]]
+
+    if khex in plugin.datastore:
+        entry = plugin.datastore[khex]
         if generation is not None:
             if entry.generation != generation:
                 raise RpcException("generation is different",
                                    DATASTORE_UPDATE_WRONG_GENERATION)
         gen = entry.generation + 1
     else:
+        # Make sure child doesn't exist (grossly inefficient)
+        if any([hex_to_key(k)[:len(key)] == key for k in plugin.datastore]):
+            raise RpcException("Key has children",
+                               DATASTORE_UPDATE_HAS_CHILDREN)
         gen = 0
 
-    plugin.datastore[key] = Entry(gen, data)
+    plugin.datastore[khex] = Entry(gen, data)
-    return datastore_entry(key, plugin.datastore[key])
+    return datastore_entry(key, plugin.datastore[khex])
 
 
 @plugin.method("deldatastore")
 def deldatastore(plugin, key, generation=None):
     """Remove a {key} from the data store"""
 
-    if not key in plugin.datastore:
+    key = normalize_key(key)
+    khex = key_to_hex(key)
+
+    if khex not in plugin.datastore:
         raise RpcException("does not exist", DATASTORE_DEL_DOES_NOT_EXIST)
 
-    entry = plugin.datastore[key]
+    entry = plugin.datastore[khex]
     if generation is not None and entry.generation != generation:
         raise RpcException("generation is different",
                            DATASTORE_DEL_WRONG_GENERATION)
 
     ret = datastore_entry(key, entry)
-    del plugin.datastore[key]
+    del plugin.datastore[khex]
     return ret
 
 
 @plugin.method("listdatastore")
-def listdatastore(plugin, key=None):
+def listdatastore(plugin, key=[]):
     """List datastore entries"""
-    if key is None:
+
-        return {'datastore': [datastore_entry(k, e)
+    key = normalize_key(key)
-                              for k, e in plugin.datastore.items()]}
+    ret = []
-    if key in plugin.datastore:
+    prev = None
-        return {'datastore': [datastore_entry(key, plugin.datastore[key])]}
+    for khex, e in sorted(plugin.datastore.items()):
-    return {'datastore': []}
+        k = hex_to_key(khex)
+        print("... {}".format(k))
+        if k[:len(key)] != key:
+            print("{} not equal".format(k[:len(key)]))
+            continue
+
+        # Don't print sub-children
+        if len(k) > len(key) + 1:
+            print("too long")
+            if prev is None or k[:len(key)+1] != prev:
+                prev = k[:len(key)+1]
+                ret += [datastore_entry(prev, None)]
+        else:
+            ret += [datastore_entry(k, e)]
+
+    return {'datastore': ret}
 
 
 def upgrade_store(plugin):
@@ -124,7 +185,7 @@ def upgrade_store(plugin):
     plugin.log("Upgrading store to have generation numbers", level='unusual')
     datastore = shelve.open('datastore_v1.dat', 'c')
     for k, d in oldstore.items():
-        datastore[k] = Entry(0, d)
+        datastore[key_to_hex([k])] = Entry(0, d)
     oldstore.close()
     datastore.close()
     os.unlink('datastore.dat')

datastore/datastore.py

@@ -2,7 +2,6 @@
 from pyln.client import Plugin, RpcError
 import shelve
 import os
-from collections import namedtuple
 
 
 plugin = Plugin()
@@ -16,9 +15,9 @@ def unload_store(plugin):
         return
 
     plugin.log("Emptying store into main store (resetting generations!)", level='unusual')
-    for k, (g, d) in datastore.items():
+    for k, (g, data) in datastore.items():
         try:
-            plugin.rpc.datastore(k, d.hex())
+            plugin.rpc.datastore(key=[k], hex=d.hex())
         except RpcError as e:
             plugin.log("Failed to put {} into store: {}".format(k, e),
                        level='broken')

datastore/test_datastore.py

@@ -1,12 +1,14 @@
 import os
 import shelve
 import time
+import pytest
 from pyln.testing.fixtures import *  # noqa: F401,F403
 from pyln.client import RpcError
 from pyln.testing.utils import only_one, wait_for
 
 plugin_path = os.path.join(os.path.dirname(__file__), "datastore.py")
 
+
 # Test taken from lightning/tests/test_misc.py
 def test_datastore(node_factory):
     l1 = node_factory.get_node(options={'plugin': plugin_path})
@@ -18,7 +20,7 @@ def test_datastore(node_factory):
 
     # Add entries.
     somedata = b'somedata'.hex()
-    somedata_expect = {'key': 'somekey',
+    somedata_expect = {'key': ['somekey'],
                        'generation': 0,
                        'hex': somedata,
                        'string': 'somedata'}
@@ -56,7 +58,7 @@ def test_datastore(node_factory):
         l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-append")
 
     otherdata = b'otherdata'.hex()
-    otherdata_expect = {'key': 'otherkey',
+    otherdata_expect = {'key': ['otherkey'],
                         'generation': 0,
                         'hex': otherdata,
                         'string': 'otherdata'}
@@ -79,7 +81,7 @@ def test_datastore(node_factory):
     assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
 
     # if it's not a string, won't print
-    badstring_expect = {'key': 'badstring',
+    badstring_expect = {'key': ['badstring'],
                         'generation': 0,
                         'hex': '00'}
     assert l1.rpc.datastore(key='badstring', hex='00') == badstring_expect
@@ -136,19 +138,74 @@ def test_upgrade(node_factory):
                                                      'datastore.dat')))
 
     vals = l1.rpc.listdatastore()['datastore']
-    assert (vals == [{'key': 'foo',
+    assert vals == [{'key': ['bar'],
-                      'generation': 0,
-                      'hex': b'foodata'.hex(),
-                      'string': 'foodata'},
-                     {'key': 'bar',
-                      'generation': 0,
-                      'hex': b'bardata'.hex(),
-                      'string': 'bardata'}]
-            or vals == [{'key': 'bar',
                      'generation': 0,
                      'hex': b'bardata'.hex(),
                      'string': 'bardata'},
-                        {'key': 'foo',
+                    {'key': ['foo'],
                      'generation': 0,
                      'hex': b'foodata'.hex(),
-                         'string': 'foodata'}])
+                     'string': 'foodata'}]
+
+
+def test_datastore_keylist(node_factory):
+    l1 = node_factory.get_node(options={'plugin': plugin_path})
+    time.sleep(5)
+
+    # Starts empty
+    assert l1.rpc.listdatastore() == {'datastore': []}
+    assert l1.rpc.listdatastore(['a']) == {'datastore': []}
+    assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
+
+    # Cannot add child to existing!
+    l1.rpc.datastore(key='a', string='aval')
+    with pytest.raises(RpcError, match='1206.*Parent key \[a\] exists'):
+        l1.rpc.datastore(key=['a', 'b'], string='abval',
+                         mode='create-or-replace')
+    # Listing subkey gives DNE.
+    assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
+    l1.rpc.deldatastore(key=['a'])
+
+    # Create child key.
+    l1.rpc.datastore(key=['a', 'b'], string='abval')
+    assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
+    assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
+                                                              'generation': 0,
+                                                              'string': 'abval',
+                                                              'hex': b'abval'.hex()}]}
+
+    # Cannot create key over that
+    with pytest.raises(RpcError, match='has children'):
+        l1.rpc.datastore(key='a', string='aval', mode='create-or-replace')
+
+    # Can create another key.
+    l1.rpc.datastore(key=['a', 'b2'], string='ab2val')
+    assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
+    assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
+                                                              'string': 'abval',
+                                                              'generation': 0,
+                                                              'hex': b'abval'.hex()},
+                                                             {'key': ['a', 'b2'],
+                                                              'string': 'ab2val',
+                                                              'generation': 0,
+                                                              'hex': b'ab2val'.hex()}]}
+
+    # Can create subkey.
+    l1.rpc.datastore(key=['a', 'b3', 'c'], string='ab2val')
+    assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
+    assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
+                                                              'string': 'abval',
+                                                              'generation': 0,
+                                                              'hex': b'abval'.hex()},
+                                                             {'key': ['a', 'b2'],
+                                                              'string': 'ab2val',
+                                                              'generation': 0,
+                                                              'hex': b'ab2val'.hex()},
+                                                             {'key': ['a', 'b3']}]}
+
+    # Can update subkey
+    l1.rpc.datastore(key=['a', 'b3', 'c'], string='2', mode='must-append')
+    assert l1.rpc.listdatastore(key=['a', 'b3', 'c']) == {'datastore': [{'key': ['a', 'b3', 'c'],
+                                                                         'string': 'ab2val2',
+                                                                         'generation': 1,
+                                                                         'hex': b'ab2val2'.hex()}]}