backup: fix flake8 code nits

This commit is contained in:
Michael Schmoock
2022-12-27 13:52:45 +01:00
committed by Christian Decker
parent 7953dc0011
commit 7f18073a3a
6 changed files with 53 additions and 25 deletions

View File

@@ -1,11 +1,13 @@
import logging, os, struct import logging
import os
import struct
import shutil import shutil
import tempfile import tempfile
from typing import Iterator from typing import Iterator
from urllib.parse import urlparse from urllib.parse import urlparse
from backend import Backend, Change from backend import Backend, Change
class FileBackend(Backend): class FileBackend(Backend):
def __init__(self, destination: str, create: bool): def __init__(self, destination: str, create: bool):
self.version = None self.version = None

View File

@@ -8,6 +8,7 @@ import zlib
from backend import Change from backend import Change
class PacketType: class PacketType:
CHANGE = 0x01 CHANGE = 0x01
SNAPSHOT = 0x02 SNAPSHOT = 0x02
@@ -21,8 +22,10 @@ class PacketType:
COMPACT = 0x0a COMPACT = 0x0a
COMPACT_RES = 0x0b COMPACT_RES = 0x0b
PKT_CHANGE_TYPES = {PacketType.CHANGE, PacketType.SNAPSHOT} PKT_CHANGE_TYPES = {PacketType.CHANGE, PacketType.SNAPSHOT}
def recvall(sock: socket.socket, n: int) -> bytearray: def recvall(sock: socket.socket, n: int) -> bytearray:
'''Receive exactly n bytes from a socket.''' '''Receive exactly n bytes from a socket.'''
buf = bytearray(n) buf = bytearray(n)
@@ -35,15 +38,18 @@ def recvall(sock: socket.socket, n: int) -> bytearray:
ptr += count ptr += count
return buf return buf
def send_packet(sock: socket.socket, typ: int, payload: bytes) -> None: def send_packet(sock: socket.socket, typ: int, payload: bytes) -> None:
sock.sendall(struct.pack('!BI', typ, len(payload))) sock.sendall(struct.pack('!BI', typ, len(payload)))
sock.sendall(payload) sock.sendall(payload)
def recv_packet(sock: socket.socket) -> Tuple[int, bytes]: def recv_packet(sock: socket.socket) -> Tuple[int, bytes]:
(typ, length) = struct.unpack('!BI', recvall(sock, 5)) (typ, length) = struct.unpack('!BI', recvall(sock, 5))
payload = recvall(sock, length) payload = recvall(sock, length)
return (typ, payload) return (typ, payload)
def change_from_packet(typ, payload): def change_from_packet(typ, payload):
'''Convert a network packet to a Change object.''' '''Convert a network packet to a Change object.'''
if typ == PacketType.CHANGE: if typ == PacketType.CHANGE:
@@ -57,6 +63,7 @@ def change_from_packet(typ, payload):
return Change(version=version, snapshot=payload, transaction=None) return Change(version=version, snapshot=payload, transaction=None)
raise ValueError('Not a change (typ {})'.format(typ)) raise ValueError('Not a change (typ {})'.format(typ))
def packet_from_change(entry): def packet_from_change(entry):
'''Convert a Change object to a network packet.''' '''Convert a Change object to a network packet.'''
if entry.snapshot is None: if entry.snapshot is None:

View File

@@ -1,10 +1,13 @@
import logging, socket, struct import logging
import socket
import struct
import json import json
import sys import sys
from typing import Tuple from typing import Tuple
from backend import Backend from backend import Backend
from protocol import PacketType, recvall, PKT_CHANGE_TYPES, change_from_packet, packet_from_change, send_packet, recv_packet from protocol import PacketType, PKT_CHANGE_TYPES, change_from_packet, packet_from_change, send_packet, recv_packet
class SystemdHandler(logging.Handler): class SystemdHandler(logging.Handler):
PREFIX = { PREFIX = {
@@ -31,6 +34,7 @@ class SystemdHandler(logging.Handler):
except Exception: except Exception:
self.handleError(record) self.handleError(record)
def setup_server_logging(mode, level): def setup_server_logging(mode, level):
root_logger = logging.getLogger() root_logger = logging.getLogger()
root_logger.setLevel(level.upper()) root_logger.setLevel(level.upper())
@@ -42,6 +46,7 @@ def setup_server_logging(mode, level):
else: else:
assert(mode == 'plain') assert(mode == 'plain')
class SocketServer: class SocketServer:
def __init__(self, addr: Tuple[str, int], backend: Backend) -> None: def __init__(self, addr: Tuple[str, int], backend: Backend) -> None:
self.backend = backend self.backend = backend
@@ -63,7 +68,7 @@ class SocketServer:
while True: while True:
try: try:
(typ, payload) = self._recv_packet() (typ, payload) = self._recv_packet()
except IOError as e: except IOError:
logging.info('Connection closed') logging.info('Connection closed')
break break
if typ in PKT_CHANGE_TYPES: if typ in PKT_CHANGE_TYPES:
@@ -118,7 +123,7 @@ class SocketServer:
conn, _ = self.bind.accept() conn, _ = self.bind.accept()
try: try:
self._handle_conn(conn) self._handle_conn(conn)
except Exception as e: except Exception:
logging.exception('Got exception') logging.exception('Got exception')
finally: finally:
conn.close() conn.close()

View File

@@ -1,10 +1,15 @@
from collections import namedtuple from collections import namedtuple
import json, logging, socket, re, struct, time import json
import logging
import socket
import re
import struct
import time
from typing import Tuple, Iterator from typing import Tuple, Iterator
from urllib.parse import urlparse, parse_qs from urllib.parse import urlparse, parse_qs
from backend import Backend, Change from backend import Backend, Change
from protocol import PacketType, recvall, PKT_CHANGE_TYPES, change_from_packet, packet_from_change, send_packet, recv_packet from protocol import PacketType, PKT_CHANGE_TYPES, change_from_packet, packet_from_change, send_packet, recv_packet
# Total number of reconnection tries # Total number of reconnection tries
RECONNECT_TRIES = 5 RECONNECT_TRIES = 5
@@ -19,16 +24,21 @@ HostPortInfo = namedtuple('HostPortInfo', ['host', 'port', 'addrtype'])
SocketURLInfo = namedtuple('SocketURLInfo', ['target', 'proxytype', 'proxytarget']) SocketURLInfo = namedtuple('SocketURLInfo', ['target', 'proxytype', 'proxytarget'])
# Network address type. # Network address type.
class AddrType: class AddrType:
IPv4 = 0 IPv4 = 0
IPv6 = 1 IPv6 = 1
NAME = 2 NAME = 2
# Proxy type. Only SOCKS5 supported at the moment as this is sufficient for Tor. # Proxy type. Only SOCKS5 supported at the moment as this is sufficient for Tor.
class ProxyType: class ProxyType:
DIRECT = 0 DIRECT = 0
SOCKS5 = 1 SOCKS5 = 1
def parse_host_port(path: str) -> HostPortInfo: def parse_host_port(path: str) -> HostPortInfo:
'''Parse a host:port pair.''' '''Parse a host:port pair.'''
if path.startswith('['): # bracketed IPv6 address if path.startswith('['): # bracketed IPv6 address
@@ -46,7 +56,7 @@ def parse_host_port(path: str) -> HostPortInfo:
if eidx == -1: if eidx == -1:
raise ValueError('Port number missing.') raise ValueError('Port number missing.')
host = path[0:eidx] host = path[0:eidx]
if re.match('\d+\.\d+\.\d+\.\d+$', host): # matches IPv4 address format if re.match(r'\d+\.\d+\.\d+\.\d+$', host): # matches IPv4 address format
addrtype = AddrType.IPv4 addrtype = AddrType.IPv4
else: else:
addrtype = AddrType.NAME addrtype = AddrType.NAME
@@ -59,6 +69,7 @@ def parse_host_port(path: str) -> HostPortInfo:
return HostPortInfo(host=host, port=port, addrtype=addrtype) return HostPortInfo(host=host, port=port, addrtype=addrtype)
def parse_socket_url(destination: str) -> SocketURLInfo: def parse_socket_url(destination: str) -> SocketURLInfo:
'''Parse a socket: URL to extract the information contained in it.''' '''Parse a socket: URL to extract the information contained in it.'''
url = urlparse(destination) url = urlparse(destination)
@@ -88,6 +99,7 @@ def parse_socket_url(destination: str) -> SocketURLInfo:
return SocketURLInfo(target=target, proxytype=proxytype, proxytarget=proxytarget) return SocketURLInfo(target=target, proxytype=proxytype, proxytarget=proxytarget)
class SocketBackend(Backend): class SocketBackend(Backend):
def __init__(self, destination: str, create: bool): def __init__(self, destination: str, create: bool):
self.version = None self.version = None

View File

@@ -1,5 +1,4 @@
from backend import Backend from backend import Backend
from filebackend import FileBackend
import socketbackend import socketbackend
from flaky import flaky from flaky import flaky
from pyln.testing.fixtures import * # noqa: F401,F403 from pyln.testing.fixtures import * # noqa: F401,F403
@@ -7,6 +6,7 @@ from pyln.testing.utils import sync_blockheight
import os import os
import pytest import pytest
import subprocess import subprocess
import tempfile
plugin_dir = os.path.dirname(__file__) plugin_dir = os.path.dirname(__file__)
@@ -236,6 +236,7 @@ class DummyBackend(Backend):
def __init__(self): def __init__(self):
pass pass
def test_rewrite(): def test_rewrite():
tests = [ tests = [
( (
@@ -281,6 +282,7 @@ def test_compact(bitcoind, directory, node_factory):
tmp = tempfile.TemporaryDirectory() tmp = tempfile.TemporaryDirectory()
subprocess.check_call([cli_path, "restore", bdest, tmp.name]) subprocess.check_call([cli_path, "restore", bdest, tmp.name])
def test_parse_socket_url(): def test_parse_socket_url():
with pytest.raises(ValueError): with pytest.raises(ValueError):
# fail: invalid url scheme # fail: invalid url scheme