aljaz/plugins
1
0
Fork 0
mirror of https://github.com/aljazceru/plugins.git synced 2025-12-19 22:24:19 +01:00
Code Issues Packages Projects Releases Wiki Activity

datastore: change keys into an array.

Browse Source 
We store this internally as hex, since shelve insists on string keys.

Signed-off-by: Rusty Russell <rusty@rustcorp.com.au>
This commit is contained in:
Rusty Russell
2021-08-05 09:50:22 +09:30
committed by Christian Decker
parent 2ec769e05b
commit 0b9e6c9517
3 changed files with 172 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files

127
datastore/datastore-plugin.py
View File

@@ -6,6 +6,7 @@ from pyln.client import Plugin, RpcException
from collections import namedtuple
import os
import shelve
from typing import Optional, Sequence, List, Union
# Error codes
DATASTORE_DEL_DOES_NOT_EXIST = 1200
@@ -13,23 +14,48 @@ DATASTORE_DEL_WRONG_GENERATION = 1201
DATASTORE_UPDATE_ALREADY_EXISTS = 1202
DATASTORE_UPDATE_DOES_NOT_EXIST = 1203
DATASTORE_UPDATE_WRONG_GENERATION = 1204
DATASTORE_UPDATE_HAS_CHILDREN = 1205
DATASTORE_UPDATE_NO_CHILDREN = 1206
plugin = Plugin()
Entry = namedtuple('Entry', ['generation', 'data'])
def datastore_entry(key, entry: Entry):
# A singleton to most commands turns into a [].
def normalize_key(key: Union[Sequence[str], str]) -> List[str]:
if not isinstance(key, list) and not isinstance(key, tuple):
key = [key]
return key
# We turn list into nul-separated hexbytes for storage
def key_to_hex(key: Sequence[str]) -> str:
return b'\0'.join([bytes(k, encoding='utf8') for k in key]).hex()
def hex_to_key(hexstr: str) -> List[str]:
return [b.decode() for b in bytes.fromhex(hexstr).split(b'\0')]
def datastore_entry(key: Sequence[str], entry: Optional[Entry]):
"""Return a dict representing the entry"""
# Entry may be a simple tuple; convert
entry = Entry(*entry)
ret = {'key': key, 'generation': entry.generation, 'hex': entry.data.hex()}
if not isinstance(key, list) and not isinstance(key, tuple):
key = [key]
# FFS, Python3 seems happy with \0 in UTF-8.
if 0 not in entry.data:
try:
ret['string'] = entry.data.decode('utf8')
except UnicodeDecodeError:
pass
ret = {'key': key}
if entry is not None:
# Entry may be a simple tuple; convert
entry = Entry(*entry)
ret['generation'] = entry.generation
ret['hex'] = entry.data.hex()
# FFS, Python3 seems happy with \0 in UTF-8.
if 0 not in entry.data:
try:
ret['string'] = entry.data.decode('utf8')
except UnicodeDecodeError:
pass
return ret
@@ -38,6 +64,8 @@ def datastore(plugin, key, string=None, hex=None, mode="must-create", generation
"""Add/modify a {key} and {hex}/{string} data to the data store,
optionally insisting it be {generation}"""
key = normalize_key(key)
khex = key_to_hex(key)
if string is not None:
if hex is not None:
raise RpcException("Cannot specify both string or hex")
@@ -48,11 +76,13 @@ optionally insisting it be {generation}"""
data = bytes.fromhex(hex)
if mode == "must-create":
if key in plugin.datastore:
raise RpcException("already exists", DATASTORE_UPDATE_ALREADY_EXISTS)
if khex in plugin.datastore:
raise RpcException("already exists",
DATASTORE_UPDATE_ALREADY_EXISTS)
elif mode == "must-replace":
if key not in plugin.datastore:
raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST)
if khex not in plugin.datastore:
raise RpcException("does not exist",
DATASTORE_UPDATE_DOES_NOT_EXIST)
elif mode == "create-or-replace":
if generation is not None:
raise RpcException("generation only valid with"
@@ -62,57 +92,88 @@ optionally insisting it be {generation}"""
if generation is not None:
raise RpcException("generation only valid with"
" must-create/must-replace")
if key not in plugin.datastore:
raise RpcException("does not exist", DATASTORE_UPDATE_DOES_NOT_EXIST)
data = plugin.datastore[key].data + data
if khex not in plugin.datastore:
raise RpcException("does not exist",
DATASTORE_UPDATE_DOES_NOT_EXIST)
data = plugin.datastore[khex].data + data
elif mode == "create-or-append":
if generation is not None:
raise RpcException("generation only valid with"
" must-create/must-replace")
data = plugin.datastore.get(key, Entry(0, bytes())).data + data
data = plugin.datastore.get(khex, Entry(0, bytes())).data + data
else:
raise RpcException("invalid mode")
if key in plugin.datastore:
entry = plugin.datastore[key]
# Make sure parent doesn't exist
parent = [key[0]]
for i in range(1, len(key)):
if key_to_hex(parent) in plugin.datastore:
raise RpcException("Parent key [{}] exists".format(','.join(parent)),
DATASTORE_UPDATE_NO_CHILDREN)
parent += [key[i]]
if khex in plugin.datastore:
entry = plugin.datastore[khex]
if generation is not None:
if entry.generation != generation:
raise RpcException("generation is different",
DATASTORE_UPDATE_WRONG_GENERATION)
gen = entry.generation + 1
else:
# Make sure child doesn't exist (grossly inefficient)
if any([hex_to_key(k)[:len(key)] == key for k in plugin.datastore]):
raise RpcException("Key has children",
DATASTORE_UPDATE_HAS_CHILDREN)
gen = 0
plugin.datastore[key] = Entry(gen, data)
return datastore_entry(key, plugin.datastore[key])
plugin.datastore[khex] = Entry(gen, data)
return datastore_entry(key, plugin.datastore[khex])
@plugin.method("deldatastore")
def deldatastore(plugin, key, generation=None):
"""Remove a {key} from the data store"""
if not key in plugin.datastore:
key = normalize_key(key)
khex = key_to_hex(key)
if khex not in plugin.datastore:
raise RpcException("does not exist", DATASTORE_DEL_DOES_NOT_EXIST)
entry = plugin.datastore[key]
entry = plugin.datastore[khex]
if generation is not None and entry.generation != generation:
raise RpcException("generation is different",
DATASTORE_DEL_WRONG_GENERATION)
ret = datastore_entry(key, entry)
del plugin.datastore[key]
del plugin.datastore[khex]
return ret
@plugin.method("listdatastore")
def listdatastore(plugin, key=None):
def listdatastore(plugin, key=[]):
"""List datastore entries"""
if key is None:
return {'datastore': [datastore_entry(k, e)
for k, e in plugin.datastore.items()]}
if key in plugin.datastore:
return {'datastore': [datastore_entry(key, plugin.datastore[key])]}
return {'datastore': []}
key = normalize_key(key)
ret = []
prev = None
for khex, e in sorted(plugin.datastore.items()):
k = hex_to_key(khex)
print("... {}".format(k))
if k[:len(key)] != key:
print("{} not equal".format(k[:len(key)]))
continue
# Don't print sub-children
if len(k) > len(key) + 1:
print("too long")
if prev is None or k[:len(key)+1] != prev:
prev = k[:len(key)+1]
ret += [datastore_entry(prev, None)]
else:
ret += [datastore_entry(k, e)]
return {'datastore': ret}
def upgrade_store(plugin):
@@ -124,7 +185,7 @@ def upgrade_store(plugin):
plugin.log("Upgrading store to have generation numbers", level='unusual')
datastore = shelve.open('datastore_v1.dat', 'c')
for k, d in oldstore.items():
datastore[k] = Entry(0, d)
datastore[key_to_hex([k])] = Entry(0, d)
oldstore.close()
datastore.close()
os.unlink('datastore.dat')

5
datastore/datastore.py
View File

@@ -2,7 +2,6 @@
from pyln.client import Plugin, RpcError
import shelve
import os
from collections import namedtuple
plugin = Plugin()
@@ -16,9 +15,9 @@ def unload_store(plugin):
return
plugin.log("Emptying store into main store (resetting generations!)", level='unusual')
for k, (g, d) in datastore.items():
for k, (g, data) in datastore.items():
try:
plugin.rpc.datastore(k, d.hex())
plugin.rpc.datastore(key=[k], hex=d.hex())
except RpcError as e:
plugin.log("Failed to put {} into store: {}".format(k, e),
level='broken')

95
datastore/test_datastore.py
View File

@@ -1,12 +1,14 @@
import os
import shelve
import time
import pytest
from pyln.testing.fixtures import * # noqa: F401,F403
from pyln.client import RpcError
from pyln.testing.utils import only_one, wait_for
plugin_path = os.path.join(os.path.dirname(__file__), "datastore.py")
# Test taken from lightning/tests/test_misc.py
def test_datastore(node_factory):
l1 = node_factory.get_node(options={'plugin': plugin_path})
@@ -18,7 +20,7 @@ def test_datastore(node_factory):
# Add entries.
somedata = b'somedata'.hex()
somedata_expect = {'key': 'somekey',
somedata_expect = {'key': ['somekey'],
'generation': 0,
'hex': somedata,
'string': 'somedata'}
@@ -56,7 +58,7 @@ def test_datastore(node_factory):
l1.rpc.datastore(key='otherkey', hex=somedata, mode="must-append")
otherdata = b'otherdata'.hex()
otherdata_expect = {'key': 'otherkey',
otherdata_expect = {'key': ['otherkey'],
'generation': 0,
'hex': otherdata,
'string': 'otherdata'}
@@ -79,7 +81,7 @@ def test_datastore(node_factory):
assert l1.rpc.listdatastore() == {'datastore': [otherdata_expect]}
# if it's not a string, won't print
badstring_expect = {'key': 'badstring',
badstring_expect = {'key': ['badstring'],
'generation': 0,
'hex': '00'}
assert l1.rpc.datastore(key='badstring', hex='00') == badstring_expect
@@ -136,19 +138,74 @@ def test_upgrade(node_factory):
'datastore.dat')))
vals = l1.rpc.listdatastore()['datastore']
assert (vals == [{'key': 'foo',
'generation': 0,
'hex': b'foodata'.hex(),
'string': 'foodata'},
{'key': 'bar',
'generation': 0,
'hex': b'bardata'.hex(),
'string': 'bardata'}]
or vals == [{'key': 'bar',
'generation': 0,
'hex': b'bardata'.hex(),
'string': 'bardata'},
{'key': 'foo',
'generation': 0,
'hex': b'foodata'.hex(),
'string': 'foodata'}])
assert vals == [{'key': ['bar'],
'generation': 0,
'hex': b'bardata'.hex(),
'string': 'bardata'},
{'key': ['foo'],
'generation': 0,
'hex': b'foodata'.hex(),
'string': 'foodata'}]
def test_datastore_keylist(node_factory):
l1 = node_factory.get_node(options={'plugin': plugin_path})
time.sleep(5)
# Starts empty
assert l1.rpc.listdatastore() == {'datastore': []}
assert l1.rpc.listdatastore(['a']) == {'datastore': []}
assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
# Cannot add child to existing!
l1.rpc.datastore(key='a', string='aval')
with pytest.raises(RpcError, match='1206.*Parent key \[a\] exists'):
l1.rpc.datastore(key=['a', 'b'], string='abval',
mode='create-or-replace')
# Listing subkey gives DNE.
assert l1.rpc.listdatastore(['a', 'b']) == {'datastore': []}
l1.rpc.deldatastore(key=['a'])
# Create child key.
l1.rpc.datastore(key=['a', 'b'], string='abval')
assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
'generation': 0,
'string': 'abval',
'hex': b'abval'.hex()}]}
# Cannot create key over that
with pytest.raises(RpcError, match='has children'):
l1.rpc.datastore(key='a', string='aval', mode='create-or-replace')
# Can create another key.
l1.rpc.datastore(key=['a', 'b2'], string='ab2val')
assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
'string': 'abval',
'generation': 0,
'hex': b'abval'.hex()},
{'key': ['a', 'b2'],
'string': 'ab2val',
'generation': 0,
'hex': b'ab2val'.hex()}]}
# Can create subkey.
l1.rpc.datastore(key=['a', 'b3', 'c'], string='ab2val')
assert l1.rpc.listdatastore() == {'datastore': [{'key': ['a']}]}
assert l1.rpc.listdatastore(key=['a']) == {'datastore': [{'key': ['a', 'b'],
'string': 'abval',
'generation': 0,
'hex': b'abval'.hex()},
{'key': ['a', 'b2'],
'string': 'ab2val',
'generation': 0,
'hex': b'ab2val'.hex()},
{'key': ['a', 'b3']}]}
# Can update subkey
l1.rpc.datastore(key=['a', 'b3', 'c'], string='2', mode='must-append')
assert l1.rpc.listdatastore(key=['a', 'b3', 'c']) == {'datastore': [{'key': ['a', 'b3', 'c'],
'string': 'ab2val2',
'generation': 1,
'hex': b'ab2val2'.hex()}]}