tpersist nodes in channel_db on disk - electrum - Electrum Bitcoin wallet
(HTM) git clone https://git.parazyd.org/electrum
(DIR) Log
(DIR) Files
(DIR) Refs
(DIR) Submodules
---
(DIR) commit bc06ded4b929641214299bbae7eb404b307ccdca
(DIR) parent 5a05a92b3d631277318cb015b56f98ee36a084ee
(HTM) Author: SomberNight <somber.night@protonmail.com>
Date: Thu, 26 Jul 2018 21:08:25 +0200
persist nodes in channel_db on disk
Diffstat:
M electrum/gui/qt/channels_list.py | 19 +++++++++++--------
M electrum/lnbase.py | 35 ++-----------------------------
M electrum/lnrouter.py | 135 +++++++++++++++++++++++++++++--
M electrum/lnutil.py | 18 +++++++++++++++++-
M electrum/lnworker.py | 9 +++++----
M electrum/network.py | 1 -
M electrum/tests/test_util.py | 21 ++++++++++++++++++++-
M electrum/util.py | 20 +++++++++++++++++++-
8 files changed, 204 insertions(+), 54 deletions(-)
---
(DIR) diff --git a/electrum/gui/qt/channels_list.py b/electrum/gui/qt/channels_list.py
t@@ -64,10 +64,12 @@ class ChannelsList(MyTreeWidget):
return h
def update_status(self):
- n = len(self.parent.network.lightning_nodes)
- nc = len(self.parent.network.channel_db)
- np = len(self.parent.wallet.lnworker.peers)
- self.status.setText(_('{} peers, {} nodes, {} channels').format(np, n, nc))
+ channel_db = self.parent.network.channel_db
+ num_nodes = len(channel_db.nodes)
+ num_channels = len(channel_db)
+ num_peers = len(self.parent.wallet.lnworker.peers)
+ self.status.setText(_('{} peers, {} nodes, {} channels')
+ .format(num_peers, num_nodes, num_channels))
def new_channel_dialog(self):
lnworker = self.parent.wallet.lnworker
t@@ -116,15 +118,16 @@ class ChannelsList(MyTreeWidget):
peer = lnworker.peers.get(node_id)
if not peer:
- known = node_id in self.parent.network.lightning_nodes
+ all_nodes = self.parent.network.channel_db.nodes
+ node_info = all_nodes.get(node_id, None)
if rest is not None:
try:
host, port = rest.split(":")
except ValueError:
self.parent.show_error(_('Connection strings must be in <node_pubkey>@<host>:<port> format'))
- elif known:
- node = self.network.lightning_nodes.get(node_id)
- host, port = node['addresses'][0]
+ return
+ elif node_info:
+ host, port = node_info.addresses[0]
else:
self.parent.show_error(_('Unknown node:') + ' ' + nodeid_hex)
return
(DIR) diff --git a/electrum/lnbase.py b/electrum/lnbase.py
t@@ -29,7 +29,7 @@ from . import crypto
from .crypto import sha256
from . import constants
from . import transaction
-from .util import PrintError, bh2u, print_error, bfh, profiler, xor_bytes
+from .util import PrintError, bh2u, print_error, bfh
from .transaction import opcodes, Transaction
from .lnonion import new_onion_packet, OnionHopsDataSingle, OnionPerHop, decode_onion_error
from .lnaddr import lndecode
t@@ -428,38 +428,7 @@ class Peer(PrintError):
self.funding_signed[channel_id].put_nowait(payload)
def on_node_announcement(self, payload):
- pubkey = payload['node_id']
- signature = payload['signature']
- h = bitcoin.Hash(payload['raw'][66:])
- if not ecc.verify_signature(pubkey, signature, h):
- return False
- self.s = payload['addresses']
- def read(n):
- data, self.s = self.s[0:n], self.s[n:]
- return data
- addresses = []
- while self.s:
- atype = ord(read(1))
- if atype == 0:
- pass
- elif atype == 1:
- ipv4_addr = '.'.join(map(lambda x: '%d' % x, read(4)))
- port = int.from_bytes(read(2), 'big')
- x = ipv4_addr, port, binascii.hexlify(pubkey)
- addresses.append((ipv4_addr, port))
- elif atype == 2:
- ipv6_addr = b':'.join([binascii.hexlify(read(2)) for i in range(4)])
- port = int.from_bytes(read(2), 'big')
- addresses.append((ipv6_addr, port))
- else:
- pass
- continue
- alias = payload['alias'].rstrip(b'\x00')
- self.network.lightning_nodes[pubkey] = {
- 'alias': alias,
- 'addresses': addresses
- }
- #self.print_error('node announcement', binascii.hexlify(pubkey), alias, addresses)
+ self.channel_db.on_node_announcement(payload)
self.network.trigger_callback('ln_status')
def on_init(self, payload):
(DIR) diff --git a/electrum/lnrouter.py b/electrum/lnrouter.py
t@@ -29,17 +29,31 @@ import json
import threading
from collections import namedtuple, defaultdict
from typing import Sequence, Union, Tuple, Optional
-
+import binascii
+import base64
from . import constants
-from .util import PrintError, bh2u, profiler, get_headers_dir, bfh
+from .util import PrintError, bh2u, profiler, get_headers_dir, bfh, is_ip_address, list_enabled_bits
from .storage import JsonDB
from .lnchanannverifier import LNChanAnnVerifier, verify_sig_for_channel_update
+from .crypto import Hash
+from . import ecc
+from .lnutil import LN_GLOBAL_FEATURE_BITS
+
+
+class UnknownEvenFeatureBits(Exception): pass
class ChannelInfo(PrintError):
def __init__(self, channel_announcement_payload):
+ self.features_len = channel_announcement_payload['len']
+ self.features = channel_announcement_payload['features']
+ enabled_features = list_enabled_bits(int.from_bytes(self.features, "big"))
+ for fbit in enabled_features:
+ if fbit not in LN_GLOBAL_FEATURE_BITS and fbit % 2 == 0:
+ raise UnknownEvenFeatureBits()
+
self.channel_id = channel_announcement_payload['short_channel_id']
self.node_id_1 = channel_announcement_payload['node_id_1']
self.node_id_2 = channel_announcement_payload['node_id_2']
t@@ -47,8 +61,6 @@ class ChannelInfo(PrintError):
assert type(self.node_id_2) is bytes
assert list(sorted([self.node_id_1, self.node_id_2])) == [self.node_id_1, self.node_id_2]
- self.features_len = channel_announcement_payload['len']
- self.features = channel_announcement_payload['features']
self.bitcoin_key_1 = channel_announcement_payload['bitcoin_key_1']
self.bitcoin_key_2 = channel_announcement_payload['bitcoin_key_2']
t@@ -162,6 +174,86 @@ class ChannelInfoDirectedPolicy:
return ChannelInfoDirectedPolicy(d2)
+class NodeInfo(PrintError):
+
+ def __init__(self, node_announcement_payload, addresses_already_parsed=False):
+ self.pubkey = node_announcement_payload['node_id']
+ self.features_len = node_announcement_payload['flen']
+ self.features = node_announcement_payload['features']
+ enabled_features = list_enabled_bits(int.from_bytes(self.features, "big"))
+ for fbit in enabled_features:
+ if fbit not in LN_GLOBAL_FEATURE_BITS and fbit % 2 == 0:
+ raise UnknownEvenFeatureBits()
+ if not addresses_already_parsed:
+ self.addresses = self.parse_addresses_field(node_announcement_payload['addresses'])
+ else:
+ self.addresses = node_announcement_payload['addresses']
+ self.alias = node_announcement_payload['alias'].rstrip(b'\x00')
+ self.timestamp = int.from_bytes(node_announcement_payload['timestamp'], "big")
+
+ @classmethod
+ def parse_addresses_field(cls, addresses_field):
+ buf = addresses_field
+ def read(n):
+ nonlocal buf
+ data, buf = buf[0:n], buf[n:]
+ return data
+ addresses = []
+ while buf:
+ atype = ord(read(1))
+ if atype == 0:
+ pass
+ elif atype == 1: # IPv4
+ ipv4_addr = '.'.join(map(lambda x: '%d' % x, read(4)))
+ port = int.from_bytes(read(2), 'big')
+ if is_ip_address(ipv4_addr) and port != 0:
+ addresses.append((ipv4_addr, port))
+ elif atype == 2: # IPv6
+ ipv6_addr = b':'.join([binascii.hexlify(read(2)) for i in range(8)])
+ ipv6_addr = ipv6_addr.decode('ascii')
+ port = int.from_bytes(read(2), 'big')
+ if is_ip_address(ipv6_addr) and port != 0:
+ addresses.append((ipv6_addr, port))
+ elif atype == 3: # onion v2
+ host = base64.b32encode(read(10)) + b'.onion'
+ host = host.decode('ascii').lower()
+ port = int.from_bytes(read(2), 'big')
+ addresses.append((host, port))
+ elif atype == 4: # onion v3
+ host = base64.b32encode(read(35)) + b'.onion'
+ host = host.decode('ascii').lower()
+ port = int.from_bytes(read(2), 'big')
+ addresses.append((host, port))
+ else:
+ # unknown address type
+ # we don't know how long it is -> have to escape
+ # if there are other addresses we could have parsed later, they are lost.
+ break
+ return addresses
+
+ def to_json(self) -> dict:
+ d = {}
+ d['node_id'] = bh2u(self.pubkey)
+ d['flen'] = bh2u(self.features_len)
+ d['features'] = bh2u(self.features)
+ d['addresses'] = self.addresses
+ d['alias'] = bh2u(self.alias)
+ d['timestamp'] = self.timestamp
+ return d
+
+ @classmethod
+ def from_json(cls, d: dict):
+ if d is None: return None
+ d2 = {}
+ d2['node_id'] = bfh(d['node_id'])
+ d2['flen'] = bfh(d['flen'])
+ d2['features'] = bfh(d['features'])
+ d2['addresses'] = d['addresses']
+ d2['alias'] = bfh(d['alias'])
+ d2['timestamp'] = d['timestamp'].to_bytes(4, "big")
+ return NodeInfo(d2, addresses_already_parsed=True)
+
+
class ChannelDB(JsonDB):
def __init__(self, network):
t@@ -173,6 +265,7 @@ class ChannelDB(JsonDB):
self.lock = threading.Lock()
self._id_to_channel_info = {}
self._channels_for_node = defaultdict(set) # node -> set(short_channel_id)
+ self.nodes = {} # node_id -> NodeInfo
self.ca_verifier = LNChanAnnVerifier(network, self)
self.network.add_jobs([self.ca_verifier])
t@@ -184,21 +277,35 @@ class ChannelDB(JsonDB):
with open(self.path, "r", encoding='utf-8') as f:
raw = f.read()
self.data = json.loads(raw)
+ # channels
channel_infos = self.get('channel_infos', {})
for short_channel_id, channel_info_d in channel_infos.items():
channel_info = ChannelInfo.from_json(channel_info_d)
short_channel_id = bfh(short_channel_id)
self.add_verified_channel_info(short_channel_id, channel_info)
+ # nodes
+ node_infos = self.get('node_infos', {})
+ for node_id, node_info_d in node_infos.items():
+ node_info = NodeInfo.from_json(node_info_d)
+ node_id = bfh(node_id)
+ self.nodes[node_id] = node_info
def save_data(self):
with self.lock:
+ # channels
channel_infos = {}
for short_channel_id, channel_info in self._id_to_channel_info.items():
channel_infos[bh2u(short_channel_id)] = channel_info
self.put('channel_infos', channel_infos)
+ # nodes
+ node_infos = {}
+ for node_id, node_info in self.nodes.items():
+ node_infos[bh2u(node_id)] = node_info
+ self.put('node_infos', node_infos)
self.write()
def __len__(self):
+ # number of channels
return len(self._id_to_channel_info)
def get_channel_info(self, channel_id) -> Optional[ChannelInfo]:
t@@ -220,7 +327,10 @@ class ChannelDB(JsonDB):
return
if constants.net.rev_genesis_bytes() != msg_payload['chain_hash']:
return
- channel_info = ChannelInfo(msg_payload)
+ try:
+ channel_info = ChannelInfo(msg_payload)
+ except UnknownEvenFeatureBits:
+ return
if trusted:
self.add_verified_channel_info(short_channel_id, channel_info)
else:
t@@ -244,6 +354,21 @@ class ChannelDB(JsonDB):
return
channel_info.on_channel_update(msg_payload, trusted=trusted)
+ def on_node_announcement(self, msg_payload):
+ pubkey = msg_payload['node_id']
+ signature = msg_payload['signature']
+ h = Hash(msg_payload['raw'][66:])
+ if not ecc.verify_signature(pubkey, signature, h):
+ return
+ old_node_info = self.nodes.get(pubkey, None)
+ try:
+ new_node_info = NodeInfo(msg_payload)
+ except UnknownEvenFeatureBits:
+ return
+ if old_node_info and old_node_info.timestamp >= new_node_info.timestamp:
+ return # ignore
+ self.nodes[pubkey] = new_node_info
+
def remove_channel(self, short_channel_id):
try:
channel_info = self._id_to_channel_info[short_channel_id]
(DIR) diff --git a/electrum/lnutil.py b/electrum/lnutil.py
t@@ -1,4 +1,4 @@
-from .util import bfh, bh2u
+from .util import bfh, bh2u, inv_dict
from .crypto import sha256
import json
from collections import namedtuple
t@@ -380,3 +380,19 @@ def overall_weight(num_htlc):
def get_ecdh(priv: bytes, pub: bytes) -> bytes:
pt = ECPubkey(pub) * string_to_number(priv)
return sha256(pt.get_public_key_bytes())
+
+
+LN_LOCAL_FEATURE_BITS = {
+ 0: 'option_data_loss_protect_req',
+ 1: 'option_data_loss_protect_opt',
+ 3: 'initial_routing_sync',
+ 4: 'option_upfront_shutdown_script_req',
+ 5: 'option_upfront_shutdown_script_opt',
+ 6: 'gossip_queries_req',
+ 7: 'gossip_queries_opt',
+}
+LN_LOCAL_FEATURE_BITS_INV = inv_dict(LN_LOCAL_FEATURE_BITS)
+
+LN_GLOBAL_FEATURE_BITS = {}
+LN_GLOBAL_FEATURE_BITS_INV = inv_dict(LN_GLOBAL_FEATURE_BITS)
+
(DIR) diff --git a/electrum/lnworker.py b/electrum/lnworker.py
t@@ -228,11 +228,12 @@ class LNWorker(PrintError):
self.peers.pop(k)
if len(self.peers) > 3:
continue
- if not self.network.lightning_nodes:
+ if not self.network.channel_db.nodes:
continue
- node_id = random.choice(list(self.network.lightning_nodes.keys()))
- node = self.network.lightning_nodes.get(node_id)
- addresses = node.get('addresses')
+ all_nodes = self.network.channel_db.nodes
+ node_id = random.choice(list(all_nodes))
+ node = all_nodes.get(node_id)
+ addresses = node.addresses
if addresses:
host, port = addresses[0]
self.print_error("trying node", bh2u(node_id))
(DIR) diff --git a/electrum/network.py b/electrum/network.py
t@@ -300,7 +300,6 @@ class Network(Logger):
self._set_status('disconnected')
# lightning network
- self.lightning_nodes = {}
self.channel_db = lnrouter.ChannelDB(self)
self.path_finder = lnrouter.LNPathFinder(self.channel_db)
self.lnwatcher = lnwatcher.LNWatcher(self)
(DIR) diff --git a/electrum/tests/test_util.py b/electrum/tests/test_util.py
t@@ -1,7 +1,7 @@
from decimal import Decimal
from electrum.util import (format_satoshis, format_fee_satoshis, parse_URI,
- is_hash256_str, chunks)
+ is_hash256_str, chunks, is_ip_address, list_enabled_bits)
from . import SequentialTestCase
t@@ -110,3 +110,22 @@ class TestUtil(SequentialTestCase):
list(chunks([1, 2, 3, 4, 5], 2)))
with self.assertRaises(ValueError):
list(chunks([1, 2, 3], 0))
+
+ def test_list_enabled_bits(self):
+ self.assertEqual((0, 2, 3, 6), list_enabled_bits(77))
+ self.assertEqual((), list_enabled_bits(0))
+
+ def test_is_ip_address(self):
+ self.assertTrue(is_ip_address("127.0.0.1"))
+ self.assertTrue(is_ip_address("127.000.000.1"))
+ self.assertTrue(is_ip_address("255.255.255.255"))
+ self.assertFalse(is_ip_address("255.255.256.255"))
+ self.assertFalse(is_ip_address("123.456.789.000"))
+ self.assertTrue(is_ip_address("2001:0db8:0000:0000:0000:ff00:0042:8329"))
+ self.assertTrue(is_ip_address("2001:db8:0:0:0:ff00:42:8329"))
+ self.assertTrue(is_ip_address("2001:db8::ff00:42:8329"))
+ self.assertFalse(is_ip_address("2001:::db8::ff00:42:8329"))
+ self.assertTrue(is_ip_address("::1"))
+ self.assertFalse(is_ip_address("2001:db8:0:0:g:ff00:42:8329"))
+ self.assertFalse(is_ip_address("lol"))
+ self.assertFalse(is_ip_address(":@ASD:@AS\x77\x22\xff¬!"))
(DIR) diff --git a/electrum/util.py b/electrum/util.py
t@@ -23,7 +23,7 @@
import binascii
import os, sys, re, json
from collections import defaultdict, OrderedDict
-from typing import NamedTuple, Union, TYPE_CHECKING, Tuple, Optional, Callable, Any
+from typing import NamedTuple, Union, TYPE_CHECKING, Tuple, Optional, Callable, Any, Sequence
from datetime import datetime
import decimal
from decimal import Decimal
t@@ -40,6 +40,7 @@ import json
import time
from typing import NamedTuple, Optional
import ssl
+import ipaddress
import aiohttp
from aiohttp_socks import SocksConnector, SocksVer
t@@ -1156,3 +1157,20 @@ def multisig_type(wallet_type):
if match:
match = [int(x) for x in match.group(1, 2)]
return match
+
+
+def is_ip_address(x: Union[str, bytes]) -> bool:
+ if isinstance(x, bytes):
+ x = x.decode("utf-8")
+ try:
+ ipaddress.ip_address(x)
+ return True
+ except ValueError:
+ return False
+
+
+def list_enabled_bits(x: int) -> Sequence[int]:
+ """e.g. 77 (0b1001101) --> (0, 2, 3, 6)"""
+ binary = bin(x)[2:]
+ rev_bin = reversed(binary)
+ return tuple(i for i, b in enumerate(rev_bin) if b == '1')