From 0b9e6c95177a850b453733c6b6d50e79b726e98f Mon Sep 17 00:00:00 2001 From: Rusty Russell Date: Thu, 5 Aug 2021 09:50:22 +0930 Subject: [PATCH] datastore: change keys into an array. We store this internally as hex, since shelve insists on string keys. Signed-off-by: Rusty Russell --- datastore/datastore-plugin.py | 127 +++++++++++++++++++++++++--------- datastore/datastore.py | 5 +- datastore/test_datastore.py | 95 ++++++++++++++++++++----- 3 files changed, 172 insertions(+), 55 deletions(-) diff --git a/datastore/datastore-plugin.py b/datastore/datastore-plugin.py index 629e7b2..32b6e4d 100755 --- a/datastore/datastore-plugin.py +++ b/datastore/datastore-plugin.py @@ -6,6 +6,7 @@ from pyln.client import Plugin, RpcException from collections import namedtuple import os import shelve +from typing import Optional, Sequence, List, Union # Error codes DATASTORE_DEL_DOES_NOT_EXIST = 1200 @@ -13,23 +14,48 @@ DATASTORE_DEL_WRONG_GENERATION = 1201 DATASTORE_UPDATE_ALREADY_EXISTS = 1202 DATASTORE_UPDATE_DOES_NOT_EXIST = 1203 DATASTORE_UPDATE_WRONG_GENERATION = 1204 +DATASTORE_UPDATE_HAS_CHILDREN = 1205 +DATASTORE_UPDATE_NO_CHILDREN = 1206 plugin = Plugin() Entry = namedtuple('Entry', ['generation', 'data']) -def datastore_entry(key, entry: Entry): + +# A singleton to most commands turns into a []. +def normalize_key(key: Union[Sequence[str], str]) -> List[str]: + if not isinstance(key, list) and not isinstance(key, tuple): + key = [key] + return key + +# We turn list into nul-separated hexbytes for storage +def key_to_hex(key: Sequence[str]) -> str: + return b'\0'.join([bytes(k, encoding='utf8') for k in key]).hex() + + +def hex_to_key(hexstr: str) -> List[str]: + return [b.decode() for b in bytes.fromhex(hexstr).split(b'\0')] + + +def datastore_entry(key: Sequence[str], entry: Optional[Entry]): """Return a dict representing the entry""" - # Entry may be a simple tuple; convert - entry = Entry(*entry) - ret = {'key': key, 'generation': entry.generation, 'hex': entry.data.hex()} + if not isinstance(key, list) and not isinstance(key, tuple): + key = [key] - # FFS, Python3 seems happy with \0 in UTF-8. - if 0 not in entry.data: - try: - ret['string'] = entry.data.decode('utf8') - except UnicodeDecodeError: - pass + ret = {'key': key} + + if entry is not None: + # Entry may be a simple tuple; convert + entry = Entry(*entry) + ret['generation'] = entry.generation + ret['hex'] = entry.data.hex() + + # FFS, Python3 seems happy with \0 in UTF-8. + if 0 not in entry.data: + try: + ret['string'] = entry.data.decode('utf8') + except UnicodeDecodeError: + pass return ret @@ -38,6 +64,8 @@ def datastore(plugin, key, string=None, hex=None, mode="must-create", generation """Add/modify a {key} and {hex}/{string} data to the data store, optionally insisting it be {generation}""" + key = normalize_key(key) + khex = key_to_hex(key) if string is not None: if hex is not None: raise RpcException("Cannot specify both string or hex") @@ -48,11 +76,13 @@ optionally insisting it be {generation}""" data = bytes.fromhex(hex) if mode == "must-create": - if key in plugin.datastore: - raise RpcException("already exists", DATASTORE_UPDATE_ALREADY_EXISTS) + if khex in plugin.datastore: + raise RpcException("already exists", + DATASTORE_UPDATE_ALREADY_EXISTS) elif mode == "must-replace": - if key not in plugin.datastore: - raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST) + if khex not in plugin.datastore: + raise RpcException("does not exist", + DATASTORE_UPDATE_DOES_NOT_EXIST) elif mode == "create-or-replace": if generation is not None: raise RpcException("generation only valid with" @@ -62,57 +92,88 @@ optionally insisting it be {generation}""" if generation is not None: raise RpcException("generation only valid with" " must-create/must-replace") - if key not in plugin.datastore: - raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST) - data = plugin.datastore[key].data + data + if khex not in plugin.datastore: + raise RpcException("does not exist", + DATASTORE_UPDATE_DOES_NOT_EXIST) + data = plugin.datastore[khex].data + data elif mode == "create-or-append": if generation is not None: raise RpcException("generation only valid with" " must-create/must-replace") - data = plugin.datastore.get(key, Entry(0, bytes())).data + data + data = plugin.datastore.get(khex, Entry(0, bytes())).data + data else: raise RpcException("invalid mode") - if key in plugin.datastore: - entry = plugin.datastore[key] + # Make sure parent doesn't exist + parent = [key[0]] + for i in range(1, len(key)): + if key_to_hex(parent) in plugin.datastore: + raise RpcException("Parent key [{}] exists".format(','.join(parent)), + DATASTORE_UPDATE_NO_CHILDREN) + parent += [key[i]] + + if khex in plugin.datastore: + entry = plugin.datastore[khex] if generation is not None: if entry.generation != generation: raise RpcException("generation is different", DATASTORE_UPDATE_WRONG_GENERATION) gen = entry.generation + 1 else: + # Make sure child doesn't exist (grossly inefficient) + if any([hex_to_key(k)[:len(key)] == key for k in plugin.datastore]): + raise RpcException("Key has children", + DATASTORE_UPDATE_HAS_CHILDREN) gen = 0 - plugin.datastore[key] = Entry(gen, data) - return datastore_entry(key, plugin.datastore[key]) + plugin.datastore[khex] = Entry(gen, data) + return datastore_entry(key, plugin.datastore[khex]) @plugin.method("deldatastore") def deldatastore(plugin, key, generation=None): """Remove a {key} from the data store""" - if not key in plugin.datastore: + key = normalize_key(key) + khex = key_to_hex(key) + + if khex not in plugin.datastore: raise RpcException("does not exist", DATASTORE_DEL_DOES_NOT_EXIST) - entry = plugin.datastore[key] + entry = plugin.datastore[khex] if generation is not None and entry.generation != generation: raise RpcException("generation is different", DATASTORE_DEL_WRONG_GENERATION) ret = datastore_entry(key, entry) - del plugin.datastore[key] + del plugin.datastore[khex] return ret @plugin.method("listdatastore") -def listdatastore(plugin, key=None): +def listdatastore(plugin, key=[]): """List datastore entries""" - if key is None: - return {'datastore': [datastore_entry(k, e) - for k, e in plugin.datastore.items()]} - if key in plugin.datastore: - return {'datastore': [datastore_entry(key, plugin.datastore[key])]} - return {'datastore': []} + + key = normalize_key(key) + ret = [] + prev = None + for khex, e in sorted(plugin.datastore.items()): + k = hex_to_key(khex) + print("... {}".format(k)) + if k[:len(key)] != key: + print("{} not equal".format(k[:len(key)])) + continue + + # Don't print sub-children + if len(k) > len(key) + 1: + print("too long") + if prev is None or k[:len(key)+1] != prev: + prev = k[:len(key)+1] + ret += [datastore_entry(prev, None)] + else: + ret += [datastore_entry(k, e)] + + return {'datastore': ret} def upgrade_store(plugin): @@ -124,7 +185,7 @@ def upgrade_store(plugin): plugin.log("Upgrading store to have generation numbers", level='unusual') datastore = shelve.open('datastore_v1.dat', 'c') for k, d in oldstore.items(): - datastore[k] = Entry(0, d) + datastore[key_to_hex([k])] = Entry(0, d) oldstore.close() datastore.close() os.unlink('datastore.dat') diff --git a/datastore/datastore.py b/datastore/datastore.py index 1671152..07b7316 100755 --- a/datastore/datastore.py +++ b/datastore/datastore.py @@ -2,7 +2,6 @@ from pyln.client import Plugin, RpcError import shelve import os -from collections import namedtuple plugin = Plugin() @@ -16,9 +15,9 @@ def unload_store(plugin): return plugin.log("Emptying store into main store (resetting generations!)", level='unusual') - for k, (g, d) in datastore.items(): + for k, (g, data) in datastore.items(): try: - plugin.rpc.datastore(k, d.hex()) + plugin.rpc.datastore(key=[k], hex=d.hex()) except RpcError as e: plugin.log("Failed to put {} into store: {}".format(k, e), level='broken') diff --git a/datastore/test_datastore.py b/datastore/test_datastore.py index 15d9e16..3fb3666 100644 --- a/datastore/test_datastore.py +++ b/datastore/test_datastore.py @@ -1,12 +1,14 @@ import os import shelve import time +import pytest from pyln.testing.fixtures import * # noqa: F401,F403 from pyln.client import RpcError from pyln.testing.utils import only_one, wait_for plugin_path = os.path.join(os.path.dirname(__file__), "datastore.py") + # Test taken from lightning/tests/test_misc.py def test_datastore(node_factory): l1 = node_factory.get_node(options={'plugin': plugin_path}) @@ -18,7 +20,7 @@ def test_datastore(node_factory): # Add entries. somedata = b'somedata'.hex() - somedata_expect = {'key': 'somekey', + somedata_expect = {'key': ['somekey'], 'generation': 0, 'hex': somedata, 'string': 'somedata'} @@ -56,7 +58,7 @@ def test_datastore(node_factory): l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-append") otherdata = b'otherdata'.hex() - otherdata_expect = {'key': 'otherkey', + otherdata_expect = {'key': ['otherkey'], 'generation': 0, 'hex': otherdata, 'string': 'otherdata'} @@ -79,7 +81,7 @@ def test_datastore(node_factory): assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]} # if it's not a string, won't print - badstring_expect = {'key': 'badstring', + badstring_expect = {'key': ['badstring'], 'generation': 0, 'hex': '00'} assert l1.rpc.datastore(key='badstring', hex='00') == badstring_expect @@ -136,19 +138,74 @@ def test_upgrade(node_factory): 'datastore.dat'))) vals = l1.rpc.listdatastore()['datastore'] - assert (vals == [{'key': 'foo', - 'generation': 0, - 'hex': b'foodata'.hex(), - 'string': 'foodata'}, - {'key': 'bar', - 'generation': 0, - 'hex': b'bardata'.hex(), - 'string': 'bardata'}] - or vals == [{'key': 'bar', - 'generation': 0, - 'hex': b'bardata'.hex(), - 'string': 'bardata'}, - {'key': 'foo', - 'generation': 0, - 'hex': b'foodata'.hex(), - 'string': 'foodata'}]) + assert vals == [{'key': ['bar'], + 'generation': 0, + 'hex': b'bardata'.hex(), + 'string': 'bardata'}, + {'key': ['foo'], + 'generation': 0, + 'hex': b'foodata'.hex(), + 'string': 'foodata'}] + + +def test_datastore_keylist(node_factory): + l1 = node_factory.get_node(options={'plugin': plugin_path}) + time.sleep(5) + + # Starts empty + assert l1.rpc.listdatastore() == {'datastore': []} + assert l1.rpc.listdatastore(['a']) == {'datastore': []} + assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []} + + # Cannot add child to existing! + l1.rpc.datastore(key='a', string='aval') + with pytest.raises(RpcError, match='1206.*Parent key \[a\] exists'): + l1.rpc.datastore(key=['a', 'b'], string='abval', + mode='create-or-replace') + # Listing subkey gives DNE. + assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []} + l1.rpc.deldatastore(key=['a']) + + # Create child key. + l1.rpc.datastore(key=['a', 'b'], string='abval') + assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]} + assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'], + 'generation': 0, + 'string': 'abval', + 'hex': b'abval'.hex()}]} + + # Cannot create key over that + with pytest.raises(RpcError, match='has children'): + l1.rpc.datastore(key='a', string='aval', mode='create-or-replace') + + # Can create another key. + l1.rpc.datastore(key=['a', 'b2'], string='ab2val') + assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]} + assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'], + 'string': 'abval', + 'generation': 0, + 'hex': b'abval'.hex()}, + {'key': ['a', 'b2'], + 'string': 'ab2val', + 'generation': 0, + 'hex': b'ab2val'.hex()}]} + + # Can create subkey. + l1.rpc.datastore(key=['a', 'b3', 'c'], string='ab2val') + assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]} + assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'], + 'string': 'abval', + 'generation': 0, + 'hex': b'abval'.hex()}, + {'key': ['a', 'b2'], + 'string': 'ab2val', + 'generation': 0, + 'hex': b'ab2val'.hex()}, + {'key': ['a', 'b3']}]} + + # Can update subkey + l1.rpc.datastore(key=['a', 'b3', 'c'], string='2', mode='must-append') + assert l1.rpc.listdatastore(key=['a', 'b3', 'c']) == {'datastore': [{'key': ['a', 'b3', 'c'], + 'string': 'ab2val2', + 'generation': 1, + 'hex': b'ab2val2'.hex()}]}