libhsmd: First working build script for libhsmd-python

Changelog-Added: libhsmd: Added python bindings for `libhsmd`

Makefile

@@ -81,6 +81,8 @@ endif
 # (method=thread to support xdist)
 PYTEST_OPTS := -v -p no:logging $(PYTEST_OPTS)
 PYTHONPATH=$(shell pwd)/contrib/pyln-client:$(shell pwd)/contrib/pyln-testing:$(shell pwd)/contrib/pyln-proto/:$(shell pwd)/external/lnprototest:$(shell pwd)/contrib/pyln-spec/bolt1:$(shell pwd)/contrib/pyln-spec/bolt2:$(shell pwd)/contrib/pyln-spec/bolt4:$(shell pwd)/contrib/pyln-spec/bolt7
+# Collect generated python files to be excluded from lint checks
+PYTHON_GENERATED=
 
 # This is where we add new features as bitcoin adds them.
 FEATURES :=
@@ -320,6 +322,7 @@ include devtools/Makefile
 include tools/Makefile
 include plugins/Makefile
 include tests/plugins/Makefile
+include contrib/libhsmd_python/Makefile
 ifneq ($(FUZZING),0)
 	include tests/fuzz/Makefile
 endif
@@ -458,7 +461,7 @@ check-python-flake8:
 	@# E501 line too long (N > 79 characters)
 	@# E731 do not assign a lambda expression, use a def
 	@# W503: line break before binary operator
-	@flake8 --ignore=E501,E731,W503 ${PYSRC}
+	@flake8 --ignore=E501,E731,W503 --exclude $(shell echo ${PYTHON_GENERATED} | sed 's/ \+/,/g') ${PYSRC}
 
 check-pytest-pyln-proto:
 	PATH=$(PYLN_PATH) PYTHONPATH=$(PYTHONPATH) $(PYTEST) contrib/pyln-proto/tests/

common/bolt12.h

@@ -104,8 +104,7 @@ bool bolt12_check_signature(const struct tlv_field *fields,
 			    const char *messagename,
 			    const char *fieldname,
 			    const struct pubkey32 *key,
-			    const struct bip340sig *sig)
+			    const struct bip340sig *sig);
-	NO_NULL_ARGS;
 
 /* Given a tal_arr of chains, does it contain this chain? */
 bool bolt12_chains_match(const struct bitcoin_blkid *chains,

contrib/libhsmd_python/Makefile

@@ -0,0 +1,11 @@
+#!/usr/bin/make
+
+LIBHSMD_PY_GEN_FILES := contrib/libhsmd_python/swig_wrap.c \
+	contrib/libhsmd_python/libhsmd.py
+
+PYTHON_GENERATED += contrib/libhsmd_python/libhsmd.py
+
+# Swig by default generates stubs in the file's directory, which is
+# what we want.
+$(LIBHSMD_PY_GEN_FILES): contrib/libhsmd_python/swig.i $(HSMD_SRC)
+	swig -python -builtin contrib/libhsmd_python/swig.i

contrib/libhsmd_python/libhsmd.py

@@ -0,0 +1,111 @@
+# This file was automatically generated by SWIG (http://www.swig.org).
+# Version 3.0.12
+#
+# Do not make changes to this file unless you know what you are doing--modify
+# the SWIG interface file instead.
+
+from sys import version_info as _swig_python_version_info
+if _swig_python_version_info >= (2, 7, 0):
+    def swig_import_helper():
+        import importlib
+        pkg = __name__.rpartition('.')[0]
+        mname = '.'.join((pkg, '_libhsmd')).lstrip('.')
+        try:
+            return importlib.import_module(mname)
+        except ImportError:
+            return importlib.import_module('_libhsmd')
+    _libhsmd = swig_import_helper()
+    del swig_import_helper
+elif _swig_python_version_info >= (2, 6, 0):
+    def swig_import_helper():
+        from os.path import dirname
+        import imp
+        fp = None
+        try:
+            fp, pathname, description = imp.find_module('_libhsmd', [dirname(__file__)])
+        except ImportError:
+            import _libhsmd
+            return _libhsmd
+        try:
+            _mod = imp.load_module('_libhsmd', fp, pathname, description)
+        finally:
+            if fp is not None:
+                fp.close()
+        return _mod
+    _libhsmd = swig_import_helper()
+    del swig_import_helper
+else:
+    import _libhsmd
+# pull in all the attributes from _libhsmd
+if __name__.rpartition('.')[0] != '':
+    if _swig_python_version_info >= (2, 7, 0):
+        try:
+            from ._libhsmd import *
+        except ImportError:
+            from _libhsmd import *
+    else:
+        from _libhsmd import *
+else:
+    from _libhsmd import *
+del _swig_python_version_info
+
+try:
+    _swig_property = property
+except NameError:
+    pass  # Python < 2.2 doesn't have 'property'.
+
+try:
+    import builtins as __builtin__
+except ImportError:
+    import __builtin__
+
+def _swig_setattr_nondynamic(self, class_type, name, value, static=1):
+    if (name == "thisown"):
+        return self.this.own(value)
+    if (name == "this"):
+        if type(value).__name__ == 'SwigPyObject':
+            self.__dict__[name] = value
+            return
+    method = class_type.__swig_setmethods__.get(name, None)
+    if method:
+        return method(self, value)
+    if (not static):
+        if _newclass:
+            object.__setattr__(self, name, value)
+        else:
+            self.__dict__[name] = value
+    else:
+        raise AttributeError("You cannot add attributes to %s" % self)
+
+
+def _swig_setattr(self, class_type, name, value):
+    return _swig_setattr_nondynamic(self, class_type, name, value, 0)
+
+
+def _swig_getattr(self, class_type, name):
+    if (name == "thisown"):
+        return self.this.own()
+    method = class_type.__swig_getmethods__.get(name, None)
+    if method:
+        return method(self)
+    raise AttributeError("'%s' object has no attribute '%s'" % (class_type.__name__, name))
+
+
+def _swig_repr(self):
+    try:
+        strthis = "proxy of " + self.this.__repr__()
+    except __builtin__.Exception:
+        strthis = ""
+    return "<%s.%s; %s >" % (self.__class__.__module__, self.__class__.__name__, strthis,)
+
+try:
+    _object = object
+    _newclass = 1
+except __builtin__.Exception:
+    class _object:
+        pass
+    _newclass = 0
+
+# This file is compatible with both classic and new-style classes.
+
+

contrib/libhsmd_python/libhsmd_python.c

@@ -0,0 +1,74 @@
+#include <ccan/str/hex/hex.h>
+#include <libhsmd_python.h>
+
+char *init(char *hex_hsm_secret, char *network_name) {
+	const struct bip32_key_version *key_version;
+	struct secret sec;
+	u8 *response;
+	setup_locale();
+	if (sodium_init() == -1) {
+		fprintf(
+		    stderr,
+		    "Could not initialize libsodium. Maybe not enough entropy"
+		    " available ?");
+		return NULL;
+	}
+
+	wally_init(0);
+	secp256k1_ctx = wally_get_secp_context();
+
+	sodium_mlock(&sec, sizeof(sec));
+	if (!hex_decode(hex_hsm_secret, strlen(hex_hsm_secret), sec.data,
+			sizeof(sec.data))) {
+		fprintf(stderr,
+			"Expected hex_hsm_secret of length 64, got %zu\n",
+			strlen(hex_hsm_secret));
+		return NULL;
+	}
+
+	/* Look up chainparams by their name */
+	chainparams = chainparams_for_network(network_name);
+	if (chainparams == NULL) {
+		fprintf(stderr, "Could not find chainparams for network %s\n",
+			network_name);
+		return NULL;
+	}
+
+	key_version = &chainparams->bip32_key_version;
+
+	response = hsmd_init(sec, *key_version);
+	sodium_munlock(&sec, sizeof(sec));
+
+	char *res = tal_hex(NULL, response);
+	tal_free(response);
+	return res;
+}
+
+char *handle(long long cap, long long dbid, char *peer_id, char *hexmsg) {
+	const tal_t *ctx = tal_arr(NULL, u8, 0);
+	size_t res_len;
+	u8 *response, *request = tal_hexdata(ctx, hexmsg, strlen(hexmsg));
+	char *res;
+	struct hsmd_client *client;
+	struct node_id *peer = NULL;
+	printf("%llu: %s\n", cap, hexmsg);
+	if (peer_id != NULL) {
+		peer = tal(ctx, struct node_id);
+		node_id_from_hexstr(hexmsg, strlen(hexmsg), peer);
+		client = hsmd_client_new_peer(ctx, cap, dbid, peer, NULL);
+	} else {
+		client = hsmd_client_new_main(ctx, cap, NULL);
+	}
+	response = hsmd_handle_client_message(NULL, client, request);
+	printf("%s\n", tal_hex(ctx, response));
+	if (response == NULL)
+		return tal_free(ctx);
+
+	res = tal_hex(NULL, response);
+	res_len = hex_str_size(tal_bytelen(response));
+	res = malloc(res_len);
+	hex_encode(response, tal_bytelen(response), res, res_len);
+
+	tal_free(ctx);
+	return res;
+}

contrib/libhsmd_python/libhsmd_python.h

@@ -0,0 +1,8 @@
+#ifndef LIGHTNING_CONTRIB_LIBHSMD_PYTHON_LIBHSMD_PYTHON_H
+#define LIGHTNING_CONTRIB_LIBHSMD_PYTHON_LIBHSMD_PYTHON_H
+
+#include <hsmd/libhsmd.h>
+char *handle(long long cap, long long dbid, char *peer_id, char *msg);
+char *init(char *hex_hsm_secret, char *network_name);
+
+#endif /* LIGHTNING_CONTRIB_LIBHSMD_PYTHON_LIBHSMD_PYTHON_H */

contrib/libhsmd_python/setup.py

@@ -0,0 +1,185 @@
+#!/usr/bin/env python
+
+"""
+setup.py file for SWIG example
+"""
+
+from distutils.core import setup, Extension
+import os
+
+# Absolute include dirs which we will later expand to full paths.
+include_dirs = [
+    ".",
+    "ccan/",
+    "external/libbacktrace/",
+    "external/libbacktrace/",
+    "external/libsodium/src/libsodium/include/sodium/",
+    "external/libwally-core/",
+    "external/libwally-core/include/",
+    "external/libwally-core/src/",
+    "external/libwally-core/src/ccan/",
+    "external/libwally-core/src/secp256k1/",
+    "external/libwally-core/src/secp256k1/include/",
+    "external/libwally-core/src/secp256k1/src",
+]
+
+sources = [
+    "bitcoin/block.c",
+    "bitcoin/chainparams.c",
+    "bitcoin/preimage.c",
+    "bitcoin/privkey.c",
+    "bitcoin/psbt.c",
+    "bitcoin/pubkey.c",
+    "bitcoin/script.c",
+    "bitcoin/shadouble.c",
+    "bitcoin/short_channel_id.c",
+    "bitcoin/signature.c",
+    "bitcoin/tx.c",
+    "bitcoin/varint.c",
+    "ccan/ccan/autodata/autodata.c",
+    "ccan/ccan/breakpoint/breakpoint.c",
+    "ccan/ccan/crypto/hkdf_sha256/hkdf_sha256.c",
+    "ccan/ccan/crypto/hmac_sha256/hmac_sha256.c",
+    "ccan/ccan/crypto/shachain/shachain.c",
+    "ccan/ccan/crypto/siphash24/siphash24.c",
+    "ccan/ccan/err/err.c",
+    "ccan/ccan/fdpass/fdpass.c",
+    "ccan/ccan/htable/htable.c",
+    "ccan/ccan/intmap/intmap.c",
+    "ccan/ccan/io/fdpass/fdpass.c",
+    "ccan/ccan/io/io.c",
+    "ccan/ccan/io/poll.c",
+    "ccan/ccan/isaac/isaac64.c",
+    "ccan/ccan/list/list.c",
+    "ccan/ccan/noerr/noerr.c",
+    "ccan/ccan/ptr_valid/ptr_valid.c",
+    "ccan/ccan/read_write_all/read_write_all.c",
+    "ccan/ccan/str/hex/hex.c",
+    "ccan/ccan/take/take.c",
+    "ccan/ccan/tal/str/str.c",
+    "ccan/ccan/tal/tal.c",
+    "ccan/ccan/time/time.c",
+    "ccan/ccan/timer/timer.c",
+    "ccan/ccan/utf8/utf8.c",
+    "common/amount.c",
+    "common/bigsize.c",
+    "common/bip32.c",
+    "common/bolt12_merkle.c",
+    "common/channel_id.c",
+    "common/daemon.c",
+    "common/daemon_conn.c",
+    "common/derive_basepoints.c",
+    "common/hash_u5.c",
+    "common/hsm_encryption.c",
+    "common/key_derive.c",
+    "common/memleak.c",
+    "common/msg_queue.c",
+    "common/node_id.c",
+    "common/pseudorand.c",
+    "common/setup.c",
+    "common/status.c",
+    "common/status_levels.c",
+    "common/status_wire.c",
+    "common/status_wiregen.c",
+    "common/subdaemon.c",
+    "common/type_to_string.c",
+    "common/utils.c",
+    "common/utxo.c",
+    "common/version.c",
+    "contrib/libhsmd_python/shims.c",
+    "contrib/libhsmd_python/swig_wrap.c",
+    "external/libbacktrace/alloc.c",
+    "external/libbacktrace/backtrace.c",
+    "external/libbacktrace/fileline.c",
+    "external/libbacktrace/posix.c",
+    "external/libbacktrace/print.c",
+    "external/libbacktrace/simple.c",
+    "external/libbacktrace/state.c",
+    "external/libbacktrace/unknown.c",
+    "external/libsodium/src/libsodium/crypto_core/ed25519/ref10/ed25519_ref10.c",
+    "external/libsodium/src/libsodium/crypto_core/hchacha20/core_hchacha20.c",
+    "external/libsodium/src/libsodium/crypto_core/salsa/ref/core_salsa_ref.c",
+    "external/libsodium/src/libsodium/crypto_generichash/blake2b/ref/blake2b-compress-ref.c",
+    "external/libsodium/src/libsodium/crypto_generichash/blake2b/ref/blake2b-ref.c",
+    "external/libsodium/src/libsodium/crypto_generichash/blake2b/ref/generichash_blake2b.c",
+    "external/libsodium/src/libsodium/crypto_onetimeauth/poly1305/donna/poly1305_donna.c",
+    "external/libsodium/src/libsodium/crypto_onetimeauth/poly1305/onetimeauth_poly1305.c",
+    "external/libsodium/src/libsodium/crypto_pwhash/argon2/argon2-core.c",
+    "external/libsodium/src/libsodium/crypto_pwhash/argon2/argon2-fill-block-ref.c",
+    "external/libsodium/src/libsodium/crypto_pwhash/argon2/blake2b-long.c",
+    "external/libsodium/src/libsodium/crypto_scalarmult/curve25519/ref10/x25519_ref10.c",
+    "external/libsodium/src/libsodium/crypto_scalarmult/curve25519/scalarmult_curve25519.c",
+    "external/libsodium/src/libsodium/crypto_secretstream/xchacha20poly1305/secretstream_xchacha20poly1305.c",
+    "external/libsodium/src/libsodium/crypto_stream/chacha20/ref/chacha20_ref.c",
+    "external/libsodium/src/libsodium/crypto_stream/chacha20/stream_chacha20.c",
+    "external/libsodium/src/libsodium/crypto_stream/salsa20/ref/salsa20_ref.c",
+    "external/libsodium/src/libsodium/crypto_stream/salsa20/stream_salsa20.c",
+    "external/libsodium/src/libsodium/crypto_verify/sodium/verify.c",
+    "external/libsodium/src/libsodium/randombytes/randombytes.c",
+    "external/libsodium/src/libsodium/randombytes/sysrandom/randombytes_sysrandom.c",
+    "external/libsodium/src/libsodium/sodium/core.c",
+    "external/libsodium/src/libsodium/sodium/runtime.c",
+    "external/libsodium/src/libsodium/sodium/utils.c",
+    "external/libwally-core/src/base58.c",
+    "external/libwally-core/src/bip32.c",
+    "external/libwally-core/src/ccan/ccan/base64/base64.c",
+    "external/libwally-core/src/ccan/ccan/crypto/ripemd160/ripemd160.c",
+    "external/libwally-core/src/ccan/ccan/crypto/sha256/sha256.c",
+    "external/libwally-core/src/ccan/ccan/crypto/sha512/sha512.c",
+    "external/libwally-core/src/hex.c",
+    "external/libwally-core/src/hmac.c",
+    "external/libwally-core/src/internal.c",
+    "external/libwally-core/src/psbt.c",
+    "external/libwally-core/src/pullpush.c",
+    "external/libwally-core/src/script.c",
+    "external/libwally-core/src/secp256k1/src/secp256k1.c",
+    "external/libwally-core/src/sign.c",
+    "external/libwally-core/src/transaction.c",
+    "hsmd/hsmd_wiregen.c",
+    "hsmd/libhsmd.c",
+    "hsmd/libhsmd_status.c",
+    "contrib/libhsmd_python/libhsmd_python.c",
+    "wire/fromwire.c",
+    "wire/peer_wire.c",
+    "wire/peer_wiregen.c",
+    "wire/tlvstream.c",
+    "wire/towire.c",
+    "wire/wire_io.c",
+    "wire/wire_sync.c",
+]
+
+include_dirs = [os.path.abspath(os.path.join("../../", f)) for f in include_dirs] + ['.']
+sources = [os.path.abspath(os.path.join("../../", f)) for f in sources]
+
+configvars = open("../../config.vars", "r").readlines()
+configtuples = [tuple(v.strip().split("=", 1)) for v in configvars]
+
+libhsmd_module = Extension(
+    "_libhsmd",
+    libraries=["sodium"],
+    include_dirs=include_dirs,
+    define_macros=configtuples
+    + [
+        ("BUILD_ELEMENTS", "1"),
+        ("SHACHAIN_BITS", "48"),
+        ("USE_NUM_NONE", "1"),
+        ("ECMULT_WINDOW_SIZE", "15"),
+        ("ECMULT_GEN_PREC_BITS", "4"),
+        ("USE_SCALAR_INV_BUILTIN", "1"),
+        ("USE_FIELD_INV_BUILTIN", "1"),
+        ("ENABLE_MODULE_EXTRAKEYS", "1"),
+        ("ENABLE_MODULE_RECOVERY", "1"),
+        ("ENABLE_MODULE_SCHNORRSIG", "1"),
+        ("ENABLE_MODULE_ECDH", "1"),
+    ],
+    sources=sources,
+)
+
+setup(
+    name="libhsmd",
+    version="0.10.0",
+    author="SWIG Docs",
+    description="""Simple swig example from docs""",
+    ext_modules=[libhsmd_module],
+    py_modules=["libhsmd"],
+)

contrib/libhsmd_python/shims.c

@@ -0,0 +1,9 @@
+#include <stdlib.h>
+#include <stdbool.h>
+
+/* The following functions are for some reason referenced but not
+ * included in the library. We provide them with dummy implementations
+ * here. */
+bool alignment_ok(void *p) { return true; }
+void dev_disconnect_init(int fd) {}
+void CCAN_CLEAR_MEMORY(void *p, size_t len) { wally_clear(p, len); }

contrib/libhsmd_python/swig.i

@@ -0,0 +1,7 @@
+%module libhsmd
+%{
+#define SWIG_FILE_WITH_INIT
+#include "libhsmd_python.h"
+%}
+
+%include "libhsmd_python.h"