pyln: Split pylightning into multiple pyln modules

This is the first step to transition to a better organized python module
structure. Sadly we can't reuse the `pylightning` module as a namespace module
since having importable things in the top level of the namespace is not
allowed in any of the namespace variants [1], hence we just switch over to the
`pyln` namespace. The code the was under `lightning` will now be reachable
under `pyln.client` and we add the `pyln.proto` module for all the things that
are independent of talking to lightningd and can be used for protocol testing.

[1] https://packaging.python.org/guides/packaging-namespace-packages/

Signed-off-by: Christian Decker <decker.christian@gmail.com>

.gitignore

@@ -39,4 +39,7 @@ tools/headerversions
 contrib/pylightning/build/
 contrib/pylightning/dist/
 contrib/pylightning/pylightning.egg-info/
+contrib/pyln-*/build/
+contrib/pyln-*/dist/
+contrib/pyln-*/pyln_*.egg-info/
 devtools/create-gossipstore

contrib/pylightning/requirements.txt

@@ -1,2 +0,0 @@
-cryptography==2.7
-coincurve==12.0.0

contrib/pyln-client/README.md

@@ -0,0 +1,101 @@
+# pyln-client: A python client library for lightningd
+
+This package implements the Unix socket based JSON-RPC protocol that
+`lightningd` exposes to the rest of the world. It can be used to call
+arbitrary functions on the RPC interface, and serves as a basis for plugins
+written in python.
+
+
+## Installation
+
+`pyln-client` is available on `pip`:
+
+```
+pip install pyln-client
+```
+
+Alternatively you can also install the development version to get access to
+currently unreleased features by checking out the c-lightning source code and
+installing into your python3 environment:
+
+```bash
+git clone https://github.com/ElementsProject/lightning.git
+cd lightning/contrib/pyln-client
+python3 setup.py develop
+```
+
+This will add links to the library into your environment so changing the
+checked out source code will also result in the environment picking up these
+changes. Notice however that unreleased versions may change API without
+warning, so test thoroughly with the released version.
+
+## Examples
+
+
+### Using the JSON-RPC client
+```py
+"""
+Generate invoice on one daemon and pay it on the other
+"""
+from pyln.client import LightningRpc
+import random
+
+# Create two instances of the LightningRpc object using two different c-lightning daemons on your computer
+l1 = LightningRpc("/tmp/lightning1/lightning-rpc")
+l5 = LightningRpc("/tmp/lightning5/lightning-rpc")
+
+info5 = l5.getinfo()
+print(info5)
+
+# Create invoice for test payment
+invoice = l5.invoice(100, "lbl{}".format(random.random()), "testpayment")
+print(invoice)
+
+# Get route to l1
+route = l1.getroute(info5['id'], 100, 1)
+print(route)
+
+# Pay invoice
+print(l1.sendpay(route['route'], invoice['payment_hash']))
+```
+
+### Writing a plugin
+
+Plugins are programs that `lightningd` can be configured to execute alongside
+the main daemon. They allow advanced interactions with and customizations to
+the daemon.
+
+```python
+#!/usr/bin/env python3
+from pyln.client import Plugin
+
+plugin = Plugin()
+
+@plugin.method("hello")
+def hello(plugin, name="world"):
+    """This is the documentation string for the hello-function.
+
+    It gets reported as the description when registering the function
+    as a method with `lightningd`.
+
+    """
+    greeting = plugin.get_option('greeting')
+    s = '{} {}'.format(greeting, name)
+    plugin.log(s)
+    return s
+
+
+@plugin.init()
+def init(options, configuration, plugin):
+    plugin.log("Plugin helloworld.py initialized")
+
+
+@plugin.subscribe("connect")
+def on_connect(plugin, id, address):
+    plugin.log("Received connect event for peer {}".format(id))
+
+
+plugin.add_option('greeting', 'Hello', 'The greeting I should use.')
+plugin.run()
+
+```

contrib/pyln-client/pyln/client/__init__.py

@@ -0,0 +1,10 @@
+from lightning import LightningRpc, Plugin, RpcError, Millisatoshi, __version__, monkey_patch
+
+__all__ = [
+    "LightningRpc",
+    "Plugin",
+    "RpcError",
+    "Millisatoshi",
+    "__version__",
+    "monkey_patch"
+]

contrib/pyln-client/requirements.txt

@@ -0,0 +1 @@
+pylightning==0.0.7.3

contrib/pyln-client/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup
+from pyln import client
+import io
+
+
+with io.open('README.md', encoding='utf-8') as f:
+    long_description = f.read()
+
+with io.open('requirements.txt', encoding='utf-8') as f:
+    requirements = [r for r in f.read().split('\n') if len(r)]
+
+setup(name='pyln-client',
+      version=client.__version__,
+      description='Client library for lightningd',
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      url='http://github.com/ElementsProject/lightning',
+      author='Christian Decker',
+      author_email='decker.christian@gmail.com',
+      license='MIT',
+      packages=['pyln.client'],
+      scripts=[],
+      zip_safe=True,
+      install_requires=requirements)

contrib/pyln-proto/README.md

@@ -0,0 +1,30 @@
+# pyln-proto: Lightning Network protocol implementation
+
+This package implements some of the Lightning Network protocol in pure
+python. It is intended for protocol testing and some minor tooling only. It is
+not deemed secure enough to handle any amount of real funds (you have been
+warned!).
+
+
+## Installation
+
+`pyln-proto` is available on `pip`:
+
+```
+pip install pyln-proto
+```
+
+Alternatively you can also install the development version to get access to
+currently unreleased features by checking out the c-lightning source code and
+installing into your python3 environment:
+
+```bash
+git clone https://github.com/ElementsProject/lightning.git
+cd lightning/contrib/pyln-proto
+python3 setup.py develop
+```
+
+This will add links to the library into your environment so changing the
+checked out source code will also result in the environment picking up these
+changes. Notice however that unreleased versions may change API without
+warning, so test thoroughly with the released version.

contrib/pyln-proto/examples/connect.py

@@ -0,0 +1,27 @@
+#!/usr/bin/env python3
+"""Simple connect and read test
+
+Connects to a peer, performs handshake and then just prints all the messages
+it gets.
+
+"""
+
+from pyln.proto.wire import connect, PrivateKey, PublicKey
+from binascii import unhexlify, hexlify
+
+ls_privkey = PrivateKey(unhexlify(
+    b'1111111111111111111111111111111111111111111111111111111111111111'
+))
+remote_pubkey = PublicKey(unhexlify(
+    b'03b31e5bbf2cdbe115b485a2b480e70a1ef3951a0dc6df4b1232e0e56f3dce18d6'
+))
+
+lc = connect(ls_privkey, remote_pubkey, '127.0.0.1', 9375)
+
+# Send an init message, with no global features, and 0b10101010 as local
+# features.
+lc.send_message(b'\x00\x10\x00\x00\x00\x01\xaa')
+
+# Now just read whatever our peer decides to send us
+while True:
+    print(hexlify(lc.read_message()).decode('ASCII'))

contrib/pyln-proto/examples/listen.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""A simple handshake and encryption test.
+
+This script will listen on port 9736 for incoming Lightning Network protocol
+connections, perform the cryptographic handshake, send 10k small pings, and
+then exit, closing the connection. This is useful to check the correct
+rotation of send- and receive-keys in the implementation.
+
+"""
+
+
+from pyln.proto.wire import LightningServerSocket, PrivateKey
+from binascii import hexlify, unhexlify
+import time
+import threading
+
+ls_privkey = PrivateKey(unhexlify(
+    b'1111111111111111111111111111111111111111111111111111111111111111'
+))
+listener = LightningServerSocket(ls_privkey)
+print("Node ID: {}".format(ls_privkey.public_key()))
+
+listener.bind(('0.0.0.0', 9735))
+listener.listen()
+c, a = listener.accept()
+
+c.send_message(b'\x00\x10\x00\x00\x00\x01\xaa')
+print(c.read_message())
+
+num_pings = 10000
+
+
+def read_loop(c):
+    for i in range(num_pings):
+        print("Recv", i, hexlify(c.read_message()))
+
+
+t = threading.Thread(target=read_loop, args=(c,))
+t.daemon = True
+t.start()
+for i in range(num_pings):
+    m = b'\x00\x12\x00\x01\x00\x01\x00'
+    c.send_message(m)
+    print("Sent", i, hexlify(m))
+    time.sleep(0.01)
+
+t.join()

contrib/pyln-proto/pyln/proto/__init__.py

@@ -0,0 +1 @@
+__version__ = '0.0.1'

contrib/pyln-proto/pyln/proto/wire.py

@@ -0,0 +1,394 @@
+from binascii import hexlify
+from cryptography.exceptions import InvalidTag
+from cryptography.hazmat.backends import default_backend
+from cryptography.hazmat.primitives import hashes
+from cryptography.hazmat.primitives.asymmetric import ec
+from cryptography.hazmat.primitives.ciphers.aead import ChaCha20Poly1305
+from cryptography.hazmat.primitives.kdf.hkdf import HKDF
+from cryptography.hazmat.primitives import serialization
+from hashlib import sha256
+import coincurve
+import os
+import socket
+import struct
+import threading
+
+
+__all__ = [
+    'PrivateKey',
+    'PublicKey',
+    'Secret',
+    'LightningConnection',
+    'LightningServerSocket',
+    'connect'
+]
+
+
+def hkdf(ikm, salt=b"", info=b""):
+    hkdf = HKDF(
+        algorithm=hashes.SHA256(),
+        length=64,
+        salt=salt,
+        info=info,
+        backend=default_backend())
+
+    return hkdf.derive(ikm)
+
+
+def hkdf_two_keys(ikm, salt):
+    t = hkdf(ikm, salt)
+    return t[:32], t[32:]
+
+
+def ecdh(k, rk):
+    k = coincurve.PrivateKey(secret=k.rawkey)
+    rk = coincurve.PublicKey(data=rk.serializeCompressed())
+    a = k.ecdh(rk.public_key)
+    return Secret(a)
+
+
+def encryptWithAD(k, n, ad, plaintext):
+    chacha = ChaCha20Poly1305(k)
+    return chacha.encrypt(n, plaintext, ad)
+
+
+def decryptWithAD(k, n, ad, ciphertext):
+    chacha = ChaCha20Poly1305(k)
+    return chacha.decrypt(n, ciphertext, ad)
+
+
+class PrivateKey(object):
+    def __init__(self, rawkey):
+        assert len(rawkey) == 32 and isinstance(rawkey, bytes)
+        self.rawkey = rawkey
+        rawkey = int(hexlify(rawkey), base=16)
+        self.key = ec.derive_private_key(rawkey, ec.SECP256K1(),
+                                         default_backend())
+
+    def serializeCompressed(self):
+        return self.key.private_bytes(serialization.Encoding.Raw,
+                                      serialization.PrivateFormat.Raw, None)
+
+    def public_key(self):
+        return PublicKey(self.key.public_key())
+
+
+class Secret(object):
+    def __init__(self, raw):
+        assert(len(raw) == 32)
+        self.raw = raw
+
+    def __str__(self):
+        return "Secret[0x{}]".format(hexlify(self.raw).decode('ASCII'))
+
+
+class PublicKey(object):
+    def __init__(self, innerkey):
+        # We accept either 33-bytes raw keys, or an EC PublicKey as returned
+        # by cryptography.io
+        if isinstance(innerkey, bytes):
+            innerkey = ec.EllipticCurvePublicKey.from_encoded_point(
+                ec.SECP256K1(), innerkey
+            )
+
+        elif not isinstance(innerkey, ec.EllipticCurvePublicKey):
+            raise ValueError(
+                "Key must either be bytes or ec.EllipticCurvePublicKey"
+            )
+        self.key = innerkey
+
+    def serializeCompressed(self):
+        raw = self.key.public_bytes(
+            serialization.Encoding.X962,
+            serialization.PublicFormat.CompressedPoint
+        )
+        return raw
+
+    def __str__(self):
+        return "PublicKey[0x{}]".format(
+            hexlify(self.serializeCompressed()).decode('ASCII')
+        )
+
+
+def Keypair(object):
+    def __init__(self, priv, pub):
+        self.priv, self.pub = priv, pub
+
+
+class Sha256Mixer(object):
+    def __init__(self, base):
+        self.hash = sha256(base).digest()
+
+    def update(self, data):
+        h = sha256(self.hash)
+        h.update(data)
+        self.hash = h.digest()
+        return self.hash
+
+    def digest(self):
+        return self.hash
+
+    def __str__(self):
+        return "Sha256Mixer[0x{}]".format(hexlify(self.hash).decode('ASCII'))
+
+
+class LightningConnection(object):
+    def __init__(self, connection, remote_pubkey, local_privkey, is_initiator):
+        self.connection = connection
+        self.chaining_key = None
+        self.handshake_hash = None
+        self.local_privkey = local_privkey
+        self.local_pubkey = self.local_privkey.public_key()
+        self.remote_pubkey = remote_pubkey
+        self.is_initiator = is_initiator
+        self.init_handshake()
+        self.rn, self.sn = 0, 0
+        self.send_lock, self.recv_lock = threading.Lock(), threading.Lock()
+
+    @classmethod
+    def nonce(cls, n):
+        """Transforms a numeric nonce into a byte formatted one
+
+        Nonce n encoded as 32 zero bits, followed by a little-endian 64-bit
+        value. Note: this follows the Noise Protocol convention, rather than
+        our normal endian.
+        """
+        return b'\x00' * 4 + struct.pack("<Q", n)
+
+    def init_handshake(self):
+        h = sha256(b'Noise_XK_secp256k1_ChaChaPoly_SHA256').digest()
+        self.chaining_key = h
+        h = sha256(h + b'lightning').digest()
+
+        if self.is_initiator:
+            responder_pubkey = self.remote_pubkey
+        else:
+            responder_pubkey = self.local_pubkey
+        h = sha256(h + responder_pubkey.serializeCompressed()).digest()
+
+        self.handshake = {
+            'h': h,
+            'e': PrivateKey(os.urandom(32)),
+        }
+
+    def handshake_act_one_initiator(self):
+        h = Sha256Mixer(b'')
+        h.hash = self.handshake['h']
+        h.update(self.handshake['e'].public_key().serializeCompressed())
+        es = ecdh(self.handshake['e'], self.remote_pubkey)
+        t = hkdf(salt=self.chaining_key, ikm=es.raw, info=b'')
+        assert(len(t) == 64)
+        self.chaining_key, temp_k1 = t[:32], t[32:]
+        c = encryptWithAD(temp_k1, self.nonce(0), h.digest(), b'')
+        self.handshake['h'] = h.update(c)
+        pk = self.handshake['e'].public_key().serializeCompressed()
+        m = b'\x00' + pk + c
+        return m
+
+    def handshake_act_one_responder(self, m):
+        v, re, c = m[0], PublicKey(m[1:34]), m[34:]
+        if v != 0:
+            raise ValueError("Unsupported handshake version {}, only version "
+                             "0 is supported.".format(v))
+
+        h = Sha256Mixer(b'')
+        h.hash = self.handshake['h']
+        h.update(re.serializeCompressed())
+        es = ecdh(self.local_privkey, re)
+        self.handshake['re'] = re
+        t = hkdf(salt=self.chaining_key, ikm=es.raw, info=b'')
+        self.chaining_key, temp_k1 = t[:32], t[32:]
+
+        try:
+            decryptWithAD(temp_k1, self.nonce(0), h.digest(), c)
+        except InvalidTag:
+            ValueError("Verification of tag failed, remote peer doesn't know "
+                       "our node ID.")
+        h.update(c)
+        self.handshake['h'] = h.digest()
+
+    def handshake_act_two_responder(self):
+        h = Sha256Mixer(b'')
+        h.hash = self.handshake['h']
+        h.update(self.handshake['e'].public_key().serializeCompressed())
+        ee = ecdh(self.handshake['e'], self.handshake['re'])
+        t = hkdf(salt=self.chaining_key, ikm=ee.raw, info=b'')
+        assert(len(t) == 64)
+        self.chaining_key, self.temp_k2 = t[:32], t[32:]
+        c = encryptWithAD(self.temp_k2, self.nonce(0), h.digest(), b'')
+        h.update(c)
+        self.handshake['h'] = h.digest()
+        pk = self.handshake['e'].public_key().serializeCompressed()
+        m = b'\x00' + pk + c
+        return m
+
+    def handshake_act_two_initiator(self, m):
+        v, re, c = m[0], PublicKey(m[1:34]), m[34:]
+        if v != 0:
+            raise ValueError("Unsupported handshake version {}, only version "
+                             "0 is supported.".format(v))
+        self.re = re
+        h = Sha256Mixer(b'')
+        h.hash = self.handshake['h']
+        h.update(re.serializeCompressed())
+        ee = ecdh(self.handshake['e'], re)
+        self.chaining_key, self.temp_k2 = hkdf_two_keys(
+            salt=self.chaining_key, ikm=ee.raw
+        )
+        try:
+            decryptWithAD(self.temp_k2, self.nonce(0), h.digest(), c)
+        except InvalidTag:
+            ValueError("Verification of tag failed.")
+        h.update(c)
+        self.handshake['h'] = h.digest()
+
+    def handshake_act_three_initiator(self):
+        h = Sha256Mixer(b'')
+        h.hash = self.handshake['h']
+        pk = self.local_pubkey.serializeCompressed()
+        c = encryptWithAD(self.temp_k2, self.nonce(1), h.digest(), pk)
+        h.update(c)
+        se = ecdh(self.local_privkey, self.re)
+
+        self.chaining_key, self.temp_k3 = hkdf_two_keys(
+            salt=self.chaining_key, ikm=se.raw
+        )
+        t = encryptWithAD(self.temp_k3, self.nonce(0), h.digest(), b'')
+        m = b'\x00' + c + t
+        t = hkdf(salt=self.chaining_key, ikm=b'', info=b'')
+
+        self.sk, self.rk = hkdf_two_keys(salt=self.chaining_key, ikm=b'')
+        self.rn, self.sn = 0, 0
+        return m
+
+    def handshake_act_three_responder(self, m):
+        h = Sha256Mixer(b'')
+        h.hash = self.handshake['h']
+        v, c, t = m[0], m[1:50], m[50:]
+        if v != 0:
+            raise ValueError("Unsupported handshake version {}, only version "
+                             "0 is supported.".format(v))
+        rs = decryptWithAD(self.temp_k2, self.nonce(1), h.digest(), c)
+        h.update(c)
+        se = ecdh(self.handshake['e'], PublicKey(rs))
+
+        self.chaining_key, self.temp_k3 = hkdf_two_keys(
+            se.raw, self.chaining_key
+        )
+        decryptWithAD(self.temp_k3, self.nonce(0), h.digest(), t)
+        self.rn, self.sn = 0, 0
+
+        self.rk, self.sk = hkdf_two_keys(salt=self.chaining_key, ikm=b'')
+
+    def read_message(self):
+        with self.recv_lock:
+            lc = self.connection.recv(18)
+            if len(lc) != 18:
+                raise ValueError(
+                    "Short read reading the message length: 18 != {}".format(
+                        len(lc))
+                )
+            length = decryptWithAD(self.rk, self.nonce(self.rn), b'', lc)
+            length, = struct.unpack("!H", length)
+            self.rn += 1
+
+            mc = self.connection.recv(length + 16)
+            if len(mc) < length + 16:
+                raise ValueError(
+                    "Short read reading the message: {} != {}".format(
+                        length + 16, len(lc)
+                    )
+                )
+            m = decryptWithAD(self.rk, self.nonce(self.rn), b'', mc)
+            self.rn += 1
+            assert(self.rn % 2 == 0)
+            self._maybe_rotate_keys()
+
+        return m
+
+    def send_message(self, m):
+        length = struct.pack("!H", len(m))
+        with self.send_lock:
+            lc = encryptWithAD(self.sk, self.nonce(self.sn), b'', length)
+            mc = encryptWithAD(self.sk, self.nonce(self.sn + 1), b'', m)
+            self.sn += 2
+            self.connection.send(lc)
+            self.connection.send(mc)
+            assert(self.sn % 2 == 0)
+            self._maybe_rotate_keys()
+
+    def _maybe_rotate_keys(self):
+        if self.sn == 1000:
+            self.sck, self.sk = hkdf_two_keys(salt=self.sck, ikm=self.sk)
+            self.sn = 0
+        if self.rn == 1000:
+            self.rck, self.rk = hkdf_two_keys(salt=self.rck, ikm=self.rk)
+            self.rn = 0
+
+    def shake(self):
+        if self.is_initiator:
+            m = self.handshake_act_one_initiator()
+            self.connection.send(m)
+            m = self.connection.recv(50)
+            if len(m) != 50:
+                raise ValueError(
+                    "Short read from peer reading act2: 50 != {}".format(
+                        len(m))
+                )
+            self.handshake_act_two_initiator(m)
+            m = self.handshake_act_three_initiator()
+            self.connection.send(m)
+        else:
+            m = self.connection.recv(50)
+            if len(m) != 50:
+                raise ValueError(
+                    "Short read from peer reading act1: 50 != {}".format(
+                        len(m))
+                )
+            self.handshake_act_one_responder(m)
+            m = self.handshake_act_two_responder()
+            self.connection.send(m)
+            m = self.connection.recv(66)
+            if len(m) != 66:
+                raise ValueError(
+                    "Short read from peer reading act3: 66 != {}".format(
+                        len(m))
+                )
+            self.handshake_act_three_responder(m)
+
+        self.sck = self.chaining_key
+        self.rck = self.chaining_key
+
+
+class LightningServerSocket(socket.socket):
+    def __init__(self, local_privkey):
+        socket.socket.__init__(self)
+        self.local_privkey = local_privkey
+        self.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+
+    def accept(self):
+        conn, address = socket.socket.accept(self)
+        lconn = LightningConnection(
+            conn, remote_pubkey=None,
+            local_privkey=self.local_privkey,
+            is_initiator=False)
+        lconn.shake()
+        return (lconn, address)
+
+
+def connect(local_privkey, node_id, host, port=9735):
+    if isinstance(node_id, bytes) and len(node_id) == 33:
+        remote_pubkey = PublicKey(node_id)
+    elif isinstance(node_id, ec.EllipticCurvePublicKey):
+        remote_pubkey = PublicKey(node_id)
+    elif isinstance(node_id, PublicKey):
+        remote_pubkey = node_id
+    else:
+        raise ValueError(
+            "node_id must be either a 33 byte array, or a PublicKey"
+        )
+    conn = socket.create_connection((host, port))
+    lconn = LightningConnection(conn, remote_pubkey, local_privkey,
+                                is_initiator=True)
+    lconn.shake()
+    return lconn

contrib/pyln-proto/requirements.txt

@@ -0,0 +1,2 @@
+cryptography==2.7
+coincurve==12.0.0

contrib/pyln-proto/setup.py

@@ -0,0 +1,24 @@
+from setuptools import setup
+from pyln import proto
+import io
+
+
+with io.open('README.md', encoding='utf-8') as f:
+    long_description = f.read()
+
+with io.open('requirements.txt', encoding='utf-8') as f:
+    requirements = [r for r in f.read().split('\n') if len(r)]
+
+setup(name='pyln-proto',
+      version=proto.__version__,
+      description='Pure python implementation of the Lightning Network protocol',
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      url='http://github.com/ElementsProject/lightning',
+      author='Christian Decker',
+      author_email='decker.christian@gmail.com',
+      license='MIT',
+      packages=['pyln.proto'],
+      scripts=[],
+      zip_safe=True,
+      install_requires=requirements)