tChannelDB: add self.lock and make it thread-safe - electrum - Electrum Bitcoin wallet (HTM) git clone https://git.parazyd.org/electrum (DIR) Log (DIR) Files (DIR) Refs (DIR) Submodules --- (DIR) commit fd56fb918982c7afbf96267e03d7cfb2ca7fc858 (DIR) parent 1ca6f6f306a1f9f376c7519b9b80662aa39768ee (HTM) Author: SomberNight <somber.night@protonmail.com> Date: Sat, 29 Feb 2020 18:32:47 +0100 ChannelDB: add self.lock and make it thread-safe Diffstat: M electrum/channel_db.py | 71 ++++++++++++++++++++----------- 1 file changed, 46 insertions(+), 25 deletions(-) --- (DIR) diff --git a/electrum/channel_db.py b/electrum/channel_db.py t@@ -31,6 +31,7 @@ from typing import Sequence, List, Tuple, Optional, Dict, NamedTuple, TYPE_CHECK import binascii import base64 import asyncio +import threading from .sql_db import SqlDB, sql t@@ -247,17 +248,21 @@ class ChannelDB(SqlDB): def __init__(self, network: 'Network'): path = os.path.join(get_headers_dir(network.config), 'gossip_db') super().__init__(network, path, commit_interval=100) + self.lock = threading.RLock() self.num_nodes = 0 self.num_channels = 0 self._channel_updates_for_private_channels = {} # type: Dict[Tuple[bytes, bytes], dict] self.ca_verifier = LNChannelVerifier(network, self) + # initialized in load_data + # note: modify/iterate needs self.lock self._channels = {} # type: Dict[bytes, ChannelInfo] self._policies = {} # type: Dict[Tuple[bytes, bytes], Policy] # (node_id, scid) -> Policy self._nodes = {} # type: Dict[bytes, NodeInfo] # node_id -> NodeInfo # node_id -> (host, port, ts) self._addresses = defaultdict(set) # type: Dict[bytes, Set[Tuple[str, int, int]]] self._channels_for_node = defaultdict(set) # type: Dict[bytes, Set[ShortChannelID]] + self.data_loaded = asyncio.Event() self.network = network # only for callback t@@ -268,16 +273,19 @@ class ChannelDB(SqlDB): self.network.trigger_callback('channel_db', self.num_nodes, self.num_channels, self.num_policies) def get_channel_ids(self): - return set(self._channels.keys()) + with self.lock: + return set(self._channels.keys()) def add_recent_peer(self, peer: LNPeerAddr): now = int(time.time()) node_id = peer.pubkey - self._addresses[node_id].add((peer.host, peer.port, now)) + with self.lock: + self._addresses[node_id].add((peer.host, peer.port, now)) self.save_node_address(node_id, peer, now) def get_200_randomly_sorted_nodes_not_in(self, node_ids): - unshuffled = set(self._nodes.keys()) - node_ids + with self.lock: + unshuffled = set(self._nodes.keys()) - node_ids return random.sample(unshuffled, min(200, len(unshuffled))) def get_last_good_address(self, node_id) -> Optional[LNPeerAddr]: t@@ -296,8 +304,10 @@ class ChannelDB(SqlDB): # FIXME this does not reliably return "recent" peers... # Also, the list() cast over the whole dict (thousands of elements), # is really inefficient. + with self.lock: + _addresses_keys = list(self._addresses.keys()) r = [self.get_last_good_address(node_id) - for node_id in list(self._addresses.keys())[-self.NUM_MAX_RECENT_PEERS:]] + for node_id in _addresses_keys[-self.NUM_MAX_RECENT_PEERS:]] return list(reversed(r)) # note: currently channel announcements are trusted by default (trusted=True); t@@ -336,9 +346,10 @@ class ChannelDB(SqlDB): except UnknownEvenFeatureBits: return channel_info = channel_info._replace(capacity_sat=capacity_sat) - self._channels[channel_info.short_channel_id] = channel_info - self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id) - self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id) + with self.lock: + self._channels[channel_info.short_channel_id] = channel_info + self._channels_for_node[channel_info.node1_id].add(channel_info.short_channel_id) + self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id) if 'raw' in msg: self.save_channel(channel_info.short_channel_id, msg['raw']) t@@ -397,7 +408,8 @@ class ChannelDB(SqlDB): if verify: self.verify_channel_update(payload) policy = Policy.from_msg(payload) - self._policies[key] = policy + with self.lock: + self._policies[key] = policy if 'raw' in payload: self.save_policy(policy.key, payload['raw']) # t@@ -492,32 +504,38 @@ class ChannelDB(SqlDB): if node and node.timestamp >= node_info.timestamp: continue # save - self._nodes[node_id] = node_info + with self.lock: + self._nodes[node_id] = node_info if 'raw' in msg_payload: self.save_node_info(node_id, msg_payload['raw']) - for addr in node_addresses: - self._addresses[node_id].add((addr.host, addr.port, 0)) + with self.lock: + for addr in node_addresses: + self._addresses[node_id].add((addr.host, addr.port, 0)) self.save_node_addresses(node_id, node_addresses) self.logger.debug("on_node_announcement: %d/%d"%(len(new_nodes), len(msg_payloads))) self.update_counts() def get_old_policies(self, delta): + with self.lock: + _policies = self._policies.copy() now = int(time.time()) - return list(k for k, v in list(self._policies.items()) if v.timestamp <= now - delta) + return list(k for k, v in _policies.items() if v.timestamp <= now - delta) def prune_old_policies(self, delta): l = self.get_old_policies(delta) if l: for k in l: - self._policies.pop(k) + with self.lock: + self._policies.pop(k) self.delete_policy(*k) self.update_counts() self.logger.info(f'Deleting {len(l)} old policies') def get_orphaned_channels(self): - ids = set(x[1] for x in self._policies.keys()) - return list(x for x in self._channels.keys() if x not in ids) + with self.lock: + ids = set(x[1] for x in self._policies.keys()) + return list(x for x in self._channels.keys() if x not in ids) def prune_orphaned_channels(self): l = self.get_orphaned_channels() t@@ -535,10 +553,11 @@ class ChannelDB(SqlDB): self._channel_updates_for_private_channels[(start_node_id, short_channel_id)] = msg_payload def remove_channel(self, short_channel_id: ShortChannelID): - channel_info = self._channels.pop(short_channel_id, None) - if channel_info: - self._channels_for_node[channel_info.node1_id].remove(channel_info.short_channel_id) - self._channels_for_node[channel_info.node2_id].remove(channel_info.short_channel_id) + with self.lock: + channel_info = self._channels.pop(short_channel_id, None) + if channel_info: + self._channels_for_node[channel_info.node1_id].remove(channel_info.short_channel_id) + self._channels_for_node[channel_info.node2_id].remove(channel_info.short_channel_id) # delete from database self.delete_channel(short_channel_id) t@@ -571,17 +590,19 @@ class ChannelDB(SqlDB): self._channels_for_node[channel_info.node2_id].add(channel_info.short_channel_id) self.logger.info(f'load data {len(self._channels)} {len(self._policies)} {len(self._channels_for_node)}') self.update_counts() - self.count_incomplete_channels() + self.logger.info(f'semi-orphaned channels: {self.get_num_incomplete_channels()}') self.data_loaded.set() - def count_incomplete_channels(self): - out = set() - for short_channel_id, ci in self._channels.items(): + def get_num_incomplete_channels(self) -> int: + found = set() + with self.lock: + _channels = self._channels.copy() + for short_channel_id, ci in _channels.items(): p1 = self.get_policy_for_node(short_channel_id, ci.node1_id) p2 = self.get_policy_for_node(short_channel_id, ci.node2_id) if p1 is None or p2 is not None: - out.add(short_channel_id) - self.logger.info(f'semi-orphaned: {len(out)}') + found.add(short_channel_id) + return len(found) def get_policy_for_node(self, short_channel_id: bytes, node_id: bytes, *, my_channels: Dict[ShortChannelID, 'Channel'] = None) -> Optional['Policy']: