From 3e64f8558f46b6c193d4eb5de9689745e6cbea23 Mon Sep 17 00:00:00 2001 From: Carlos Ruz Date: Mon, 2 Sep 2024 15:39:06 -0600 Subject: [PATCH 1/8] Add mutinynet --- sauron/sauron.py | 148 ++++++++++++++++++++++++++++++++--------------- 1 file changed, 102 insertions(+), 46 deletions(-) diff --git a/sauron/sauron.py b/sauron/sauron.py index 3e2e34c9a..54ae05dd4 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -2,22 +2,20 @@ import requests import sys import time +from pprint import pprint -from requests.packages.urllib3.util.retry import Retry +from urllib3.util.retry import Retry from requests.adapters import HTTPAdapter from art import sauron_eye from pyln.client import Plugin - plugin = Plugin(dynamic=False) plugin.sauron_socks_proxies = None plugin.sauron_network = "test" - class SauronError(Exception): pass - def fetch(url): """Fetch this {url}, maybe through a pre-defined proxy.""" # FIXME: Maybe try to be smart and renew circuit to broadcast different @@ -38,10 +36,11 @@ def fetch(url): return session.get(url) - @plugin.init() -def init(plugin, options, configuration, **kwargs): - plugin.api_endpoint = options["sauron-api-endpoint"] +def init(plugin, options, **kwargs): + plugin.api_endpoint = options.get("sauron-api-endpoint", None) + plugin.log("plugin.api_endpoint = %s" % plugin.api_endpoint) + if not plugin.api_endpoint: raise SauronError("You need to specify the sauron-api-endpoint option.") sys.exit(1) @@ -58,7 +57,6 @@ def init(plugin, options, configuration, **kwargs): plugin.log("Sauron plugin initialized") plugin.log(sauron_eye) - @plugin.method("getchaininfo") def getchaininfo(plugin, **kwargs): blockhash_url = "{}/block-height/0".format(plugin.api_endpoint) @@ -99,7 +97,6 @@ def getchaininfo(plugin, **kwargs): "ibd": False, } - @plugin.method("getrawblockbyheight") def getrawblock(plugin, height, **kwargs): blockhash_url = "{}/block-height/{}".format(plugin.api_endpoint, height) @@ -110,7 +107,11 @@ def getrawblock(plugin, height, **kwargs): "block": None, } - block_url = "{}/block/{}/raw".format(plugin.api_endpoint, blockhash_req.text) + block_hash = blockhash_req.text.strip() # Ensure no extra spaces or newlines + + # Step 2: Determine the block URL and fetch the block data + block_url = "{}/block/{}/raw".format(plugin.api_endpoint, block_hash) + while True: block_req = fetch(block_url) if block_req.status_code != 200: @@ -130,16 +131,22 @@ def getrawblock(plugin, height, **kwargs): plugin.log("Esplora gave us an incomplete block, retrying in 2s", level="error") time.sleep(2) + plugin.log("block_req = %s" % pprint(vars(block_req))) + + # Step 3: Process the block data + # Blockstream and Mutinynet returns raw binary data + block_data = block_req.content.hex() + plugin.log("block_data = %s" % block_data) + return { - "blockhash": blockhash_req.text, - "block": block_req.content.hex(), + "blockhash": block_hash, + "block": block_data, } @plugin.method("sendrawtransaction") def sendrawtx(plugin, tx, **kwargs): sendtx_url = "{}/tx".format(plugin.api_endpoint) - sendtx_req = requests.post(sendtx_url, data=tx) if sendtx_req.status_code != 200: return { @@ -154,57 +161,107 @@ def sendrawtx(plugin, tx, **kwargs): @plugin.method("getutxout") -def getutxout(plugin, txid, vout, **kwargs): - gettx_url = "{}/tx/{}".format(plugin.api_endpoint, txid) - status_url = "{}/tx/{}/outspend/{}".format(plugin.api_endpoint, txid, vout) +def getutxout(plugin, address, txid, vout, **kwargs): + # Determine the API endpoint type based on the URL structure + if "mutinynet" in plugin.api_endpoint: + # MutinyNet API + utxo_url = "{}/address/{}/utxo".format(plugin.api_endpoint, address) + + # Fetch the list of UTXOs for the given address + utxo_req = fetch(utxo_url) + if not utxo_req.status_code == 200: + raise SauronError( + "Endpoint at {} returned {} ({}) when trying to get UTXOs.".format( + utxo_url, utxo_req.status_code, utxo_req.text + ) + ) - gettx_req = fetch(gettx_url) - if not gettx_req.status_code == 200: - raise SauronError( - "Endpoint at {} returned {} ({}) when trying to " "get transaction.".format( - gettx_url, gettx_req.status_code, gettx_req.text + # Parse the UTXO data + utxos = utxo_req.json() + # Find the UTXO with the given txid and vout + for utxo in utxos: + if utxo['txid'] == txid and utxo['vout'] == vout: + return { + "amount": utxo["value"], + "script": None # MutinyNet API does not provide script information + } + + # If the specific UTXO is not found + return { + "amount": None, + "script": None + } + + else: + # Blockstream API + gettx_url = "{}/tx/{}".format(plugin.api_endpoint, txid) + status_url = "{}/tx/{}/outspend/{}".format(plugin.api_endpoint, txid, vout) + + gettx_req = fetch(gettx_url) + if not gettx_req.status_code == 200: + raise SauronError( + "Endpoint at {} returned {} ({}) when trying to get transaction.".format( + gettx_url, gettx_req.status_code, gettx_req.text + ) ) - ) - status_req = fetch(status_url) - if not status_req.status_code == 200: - raise SauronError( - "Endpoint at {} returned {} ({}) when trying to " "get utxo status.".format( - status_url, status_req.status_code, status_req.text + status_req = fetch(status_url) + if not status_req.status_code == 200: + raise SauronError( + "Endpoint at {} returned {} ({}) when trying to get UTXO status.".format( + status_url, status_req.status_code, status_req.text + ) ) - ) - if status_req.json()["spent"]: + if status_req.json()["spent"]: + return { + "amount": None, + "script": None + } + + txo = gettx_req.json()["vout"][vout] return { - "amount": None, - "script": None, + "amount": txo["value"], + "script": txo["scriptpubkey"] } - txo = gettx_req.json()["vout"][vout] - return { - "amount": txo["value"], - "script": txo["scriptpubkey"], - } @plugin.method("estimatefees") def estimatefees(plugin, **kwargs): - feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) + # Define the URL based on the selected API + if "mutinynet" in plugin.api_endpoint: + # MutinyNet API + feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) + plugin.log("estimatefees: plugin.api_endpoint = %s" % plugin.api_endpoint) + plugin.log("estimatefees: feerate_url = %s" % feerate_url) + + else: + # Blockstream API + feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) + plugin.log("estimatefees: plugin.api_endpoint = %s" % plugin.api_endpoint) + plugin.log("estimatefees: feerate_url = %s" % feerate_url) feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 feerates = feerate_req.json() - if plugin.sauron_network == "test" or plugin.sauron_network == "signet": - # FIXME: remove the hack if the test API is "fixed" + plugin.log("estimatefees: feerates = %s" % feerates) + + # Define the multiply factor for sat/vB to sat/kVB conversion + multiply_factor = 10**3 + + if plugin.sauron_network in ["test", "signet"]: + # Apply the fallback for test/signet networks feerate = feerates.get("144", 1) - slow = normal = urgent = very_urgent = int(feerate * 10**3) + slow = normal = urgent = very_urgent = int(feerate * multiply_factor) else: + # Adjust fee rates based on the specific API # It returns sat/vB, we want sat/kVB, so multiply everything by 10**3 - slow = int(feerates["144"] * 10**3) - normal = int(feerates["12"] * 10**3) - urgent = int(feerates["6"] * 10**3) - very_urgent = int(feerates["2"] * 10**3) + slow = int(feerates["144"] * multiply_factor) + normal = int(feerates["12"] * multiply_factor) + urgent = int(feerates["6"] * multiply_factor) + very_urgent = int(feerates["2"] * multiply_factor) - feerate_floor = int(feerates["1008"] * 10**3) + feerate_floor = int(feerates.get("1008", slow) * multiply_factor) feerates = [ {"blocks": 2, "feerate": very_urgent}, {"blocks": 6, "feerate": urgent}, @@ -225,7 +282,6 @@ def estimatefees(plugin, **kwargs): "feerates": feerates } - plugin.add_option( "sauron-api-endpoint", "", From acbd990b04db79945b7f71abff490f69c121483c Mon Sep 17 00:00:00 2001 From: Jorge Martinez Ortega Date: Tue, 17 Sep 2024 13:54:40 -0600 Subject: [PATCH 2/8] test if backend is Esplora or Mempool space remove some redundant code lines --- sauron/sauron.py | 69 +++++++++++++++++++++++++++--------------------- 1 file changed, 39 insertions(+), 30 deletions(-) diff --git a/sauron/sauron.py b/sauron/sauron.py index 54ae05dd4..0884fc1c9 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -45,6 +45,24 @@ def init(plugin, options, **kwargs): raise SauronError("You need to specify the sauron-api-endpoint option.") sys.exit(1) + # Test for Esplora or mempoolspace + try: + # MutinyNet API + feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) + feerate_req = fetch(feerate_url) + assert feerate_req.status_code == 200 + plugin.is_mempoolspace = True + except AssertionError as e0: + try: + # Blockstream API + feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) + feerate_req = fetch(feerate_url) + assert feerate_req.status_code == 200 + plugin.is_mempoolspace = False + except AssertionError as e1: + raise Exception("Sauron API cannot be reached") from e1 + + if options["sauron-tor-proxy"]: address, port = options["sauron-tor-proxy"].split(":") socks5_proxy = "socks5h://{}:{}".format(address, port) @@ -163,24 +181,29 @@ def sendrawtx(plugin, tx, **kwargs): @plugin.method("getutxout") def getutxout(plugin, address, txid, vout, **kwargs): # Determine the API endpoint type based on the URL structure - if "mutinynet" in plugin.api_endpoint: + if plugin.is_mempoolspace: # MutinyNet API - utxo_url = "{}/address/{}/utxo".format(plugin.api_endpoint, address) + gettx_url = "{}/address/{}/utxo".format(plugin.api_endpoint, address) + else: + # Blockstream API + gettx_url = "{}/tx/{}".format(plugin.api_endpoint, txid) + status_url = "{}/tx/{}/outspend/{}".format(plugin.api_endpoint, txid, vout) - # Fetch the list of UTXOs for the given address - utxo_req = fetch(utxo_url) - if not utxo_req.status_code == 200: - raise SauronError( - "Endpoint at {} returned {} ({}) when trying to get UTXOs.".format( - utxo_url, utxo_req.status_code, utxo_req.text - ) + # Fetch the list of UTXOs for the given address + gettx_req = fetch(gettx_url) + if not gettx_req.status_code == 200: + raise SauronError( + "Endpoint at {} returned {} ({}) when trying to get transaction.".format( + gettx_url, gettx_req.status_code, gettx_req.text ) - + ) + if plugin.is_mempoolspace: + # Building response from MutinyNet API # Parse the UTXO data - utxos = utxo_req.json() + utxos = gettx_req.json() # Find the UTXO with the given txid and vout for utxo in utxos: - if utxo['txid'] == txid and utxo['vout'] == vout: + if utxo['txid'] == txid: return { "amount": utxo["value"], "script": None # MutinyNet API does not provide script information @@ -191,19 +214,8 @@ def getutxout(plugin, address, txid, vout, **kwargs): "amount": None, "script": None } - else: - # Blockstream API - gettx_url = "{}/tx/{}".format(plugin.api_endpoint, txid) - status_url = "{}/tx/{}/outspend/{}".format(plugin.api_endpoint, txid, vout) - - gettx_req = fetch(gettx_url) - if not gettx_req.status_code == 200: - raise SauronError( - "Endpoint at {} returned {} ({}) when trying to get transaction.".format( - gettx_url, gettx_req.status_code, gettx_req.text - ) - ) + # Building response from Blockstream API status_req = fetch(status_url) if not status_req.status_code == 200: raise SauronError( @@ -229,18 +241,15 @@ def getutxout(plugin, address, txid, vout, **kwargs): @plugin.method("estimatefees") def estimatefees(plugin, **kwargs): # Define the URL based on the selected API - if "mutinynet" in plugin.api_endpoint: + if plugin.is_mempoolspace: # MutinyNet API feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) - plugin.log("estimatefees: plugin.api_endpoint = %s" % plugin.api_endpoint) - plugin.log("estimatefees: feerate_url = %s" % feerate_url) - else: # Blockstream API feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) - plugin.log("estimatefees: plugin.api_endpoint = %s" % plugin.api_endpoint) - plugin.log("estimatefees: feerate_url = %s" % feerate_url) + plugin.log("estimatefees: plugin.api_endpoint = %s" % plugin.api_endpoint) + plugin.log("estimatefees: feerate_url = %s" % feerate_url) feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 feerates = feerate_req.json() From 652fbd6baf620aa883fe2258ac1d9ca056bb2dc9 Mon Sep 17 00:00:00 2001 From: Carlos Ruz Date: Tue, 24 Sep 2024 15:01:29 -0600 Subject: [PATCH 3/8] Update requirements.txt and README.md for use with mutinynet --- sauron/README.md | 8 ++++++++ sauron/requirements.txt | 2 +- sauron/sauron.py | 7 ++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/sauron/README.md b/sauron/README.md index d02dbe10e..ce023c808 100644 --- a/sauron/README.md +++ b/sauron/README.md @@ -24,6 +24,14 @@ Here is a fully reptilian example running against [blockstream.info](https://blo lightningd --mainnet --disable-plugin bcli --plugin $PWD/sauron.py --sauron-api-endpoint https://blockstream.info/api/ ``` + +Here is an example running against [mutinynet.com](https://mutinynet.com/): + +``` +lightningd --signet --disable-plugin bcli --plugin $PWD/sauron.py --sauron-api-endpoint https://mutinynet.com/api/ +``` + + You can use also proxy your requests through [Tor](https://www.torproject.org/) by specifying a SOCKS proxy to use with the `--sauron-tor-proxy` startup option, in the form `address:port`. diff --git a/sauron/requirements.txt b/sauron/requirements.txt index 1794cb833..f8b0c253f 100644 --- a/sauron/requirements.txt +++ b/sauron/requirements.txt @@ -1,2 +1,2 @@ -pyln-client>=23.2 +pyln-client==24.5 requests[socks]>=2.23.0 diff --git a/sauron/sauron.py b/sauron/sauron.py index 0884fc1c9..f58208e7a 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -17,7 +17,7 @@ class SauronError(Exception): pass def fetch(url): - """Fetch this {url}, maybe through a pre-defined proxy.""" + """Fetch the given {url}, maybe through a pre-defined proxy.""" # FIXME: Maybe try to be smart and renew circuit to broadcast different # transactions ? Hint: lightningd will agressively send us the same # transaction a certain amount of times. @@ -79,6 +79,7 @@ def init(plugin, options, **kwargs): def getchaininfo(plugin, **kwargs): blockhash_url = "{}/block-height/0".format(plugin.api_endpoint) blockcount_url = "{}/blocks/tip/height".format(plugin.api_endpoint) + chains = { "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f": "main", "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943": "test", @@ -117,7 +118,9 @@ def getchaininfo(plugin, **kwargs): @plugin.method("getrawblockbyheight") def getrawblock(plugin, height, **kwargs): + # Step 1: Get the block hash by height blockhash_url = "{}/block-height/{}".format(plugin.api_endpoint, height) + blockhash_req = fetch(blockhash_url) if blockhash_req.status_code != 200: return { @@ -278,6 +281,7 @@ def estimatefees(plugin, **kwargs): {"blocks": 144, "feerate": slow} ] + # Return the estimated fees return { "opening": normal, "mutual_close": normal, @@ -295,6 +299,7 @@ def estimatefees(plugin, **kwargs): "sauron-api-endpoint", "", "The URL of the esplora instance to hit (including '/api').", + "The URL of the mutinynet instance to hit (including '/api').", ) plugin.add_option( From 8cee2e490777754c80878016db9758f9c10e10ff Mon Sep 17 00:00:00 2001 From: Carlos Ruz Date: Fri, 27 Sep 2024 17:25:43 -0600 Subject: [PATCH 4/8] Addressed PR comments --- sauron/.gitignore | 1 + sauron/sauron.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 9 deletions(-) create mode 100644 sauron/.gitignore diff --git a/sauron/.gitignore b/sauron/.gitignore new file mode 100644 index 000000000..21d0b898f --- /dev/null +++ b/sauron/.gitignore @@ -0,0 +1 @@ +.venv/ diff --git a/sauron/sauron.py b/sauron/sauron.py index f58208e7a..65fb99124 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -47,22 +47,21 @@ def init(plugin, options, **kwargs): # Test for Esplora or mempoolspace try: - # MutinyNet API - feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) + # Esplora API + feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 - plugin.is_mempoolspace = True + plugin.is_mempoolspace = False except AssertionError as e0: try: - # Blockstream API - feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) + # MutinyNet API + feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 - plugin.is_mempoolspace = False + plugin.is_mempoolspace = True except AssertionError as e1: raise Exception("Sauron API cannot be reached") from e1 - if options["sauron-tor-proxy"]: address, port = options["sauron-tor-proxy"].split(":") socks5_proxy = "socks5h://{}:{}".format(address, port) @@ -298,8 +297,7 @@ def estimatefees(plugin, **kwargs): plugin.add_option( "sauron-api-endpoint", "", - "The URL of the esplora instance to hit (including '/api').", - "The URL of the mutinynet instance to hit (including '/api').", + "The URL of the esplora or mempool.space instance to hit (including '/api').", ) plugin.add_option( From 7402d7d07565f2868dc0cdbf9cf8138fbb68e524 Mon Sep 17 00:00:00 2001 From: Carlos Ruz Date: Mon, 30 Sep 2024 17:21:00 -0600 Subject: [PATCH 5/8] Update pyln-client requirements, fix bug and clean code. --- sauron/requirements.txt | 2 +- sauron/sauron.py | 124 +++++++++++++--------------------------- 2 files changed, 40 insertions(+), 86 deletions(-) diff --git a/sauron/requirements.txt b/sauron/requirements.txt index f8b0c253f..7fd22f35e 100644 --- a/sauron/requirements.txt +++ b/sauron/requirements.txt @@ -1,2 +1,2 @@ -pyln-client==24.5 +pyln-client>=23.2,<=24.5 requests[socks]>=2.23.0 diff --git a/sauron/sauron.py b/sauron/sauron.py index 65fb99124..0ac6b9375 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -2,22 +2,24 @@ import requests import sys import time -from pprint import pprint -from urllib3.util.retry import Retry +from requests.packages.urllib3.util.retry import Retry from requests.adapters import HTTPAdapter from art import sauron_eye from pyln.client import Plugin + plugin = Plugin(dynamic=False) plugin.sauron_socks_proxies = None plugin.sauron_network = "test" + class SauronError(Exception): pass + def fetch(url): - """Fetch the given {url}, maybe through a pre-defined proxy.""" + """Fetch this {url}, maybe through a pre-defined proxy.""" # FIXME: Maybe try to be smart and renew circuit to broadcast different # transactions ? Hint: lightningd will agressively send us the same # transaction a certain amount of times. @@ -36,11 +38,10 @@ def fetch(url): return session.get(url) + @plugin.init() def init(plugin, options, **kwargs): plugin.api_endpoint = options.get("sauron-api-endpoint", None) - plugin.log("plugin.api_endpoint = %s" % plugin.api_endpoint) - if not plugin.api_endpoint: raise SauronError("You need to specify the sauron-api-endpoint option.") sys.exit(1) @@ -74,11 +75,11 @@ def init(plugin, options, **kwargs): plugin.log("Sauron plugin initialized") plugin.log(sauron_eye) + @plugin.method("getchaininfo") def getchaininfo(plugin, **kwargs): blockhash_url = "{}/block-height/0".format(plugin.api_endpoint) blockcount_url = "{}/blocks/tip/height".format(plugin.api_endpoint) - chains = { "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f": "main", "000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943": "test", @@ -115,11 +116,10 @@ def getchaininfo(plugin, **kwargs): "ibd": False, } + @plugin.method("getrawblockbyheight") def getrawblock(plugin, height, **kwargs): - # Step 1: Get the block hash by height blockhash_url = "{}/block-height/{}".format(plugin.api_endpoint, height) - blockhash_req = fetch(blockhash_url) if blockhash_req.status_code != 200: return { @@ -127,11 +127,7 @@ def getrawblock(plugin, height, **kwargs): "block": None, } - block_hash = blockhash_req.text.strip() # Ensure no extra spaces or newlines - - # Step 2: Determine the block URL and fetch the block data - block_url = "{}/block/{}/raw".format(plugin.api_endpoint, block_hash) - + block_url = "{}/block/{}/raw".format(plugin.api_endpoint, blockhash_req.text) while True: block_req = fetch(block_url) if block_req.status_code != 200: @@ -151,22 +147,16 @@ def getrawblock(plugin, height, **kwargs): plugin.log("Esplora gave us an incomplete block, retrying in 2s", level="error") time.sleep(2) - plugin.log("block_req = %s" % pprint(vars(block_req))) - - # Step 3: Process the block data - # Blockstream and Mutinynet returns raw binary data - block_data = block_req.content.hex() - plugin.log("block_data = %s" % block_data) - return { - "blockhash": block_hash, - "block": block_data, + "blockhash": blockhash_req.text, + "block": block_req.content.hex(), } @plugin.method("sendrawtransaction") def sendrawtx(plugin, tx, **kwargs): sendtx_url = "{}/tx".format(plugin.api_endpoint) + sendtx_req = requests.post(sendtx_url, data=tx) if sendtx_req.status_code != 200: return { @@ -181,68 +171,40 @@ def sendrawtx(plugin, tx, **kwargs): @plugin.method("getutxout") -def getutxout(plugin, address, txid, vout, **kwargs): - # Determine the API endpoint type based on the URL structure - if plugin.is_mempoolspace: - # MutinyNet API - gettx_url = "{}/address/{}/utxo".format(plugin.api_endpoint, address) - else: - # Blockstream API - gettx_url = "{}/tx/{}".format(plugin.api_endpoint, txid) - status_url = "{}/tx/{}/outspend/{}".format(plugin.api_endpoint, txid, vout) +def getutxout(plugin, txid, vout, **kwargs): + gettx_url = "{}/tx/{}".format(plugin.api_endpoint, txid) + status_url = "{}/tx/{}/outspend/{}".format(plugin.api_endpoint, txid, vout) - # Fetch the list of UTXOs for the given address gettx_req = fetch(gettx_url) if not gettx_req.status_code == 200: raise SauronError( - "Endpoint at {} returned {} ({}) when trying to get transaction.".format( + "Endpoint at {} returned {} ({}) when trying to " "get transaction.".format( gettx_url, gettx_req.status_code, gettx_req.text ) ) - if plugin.is_mempoolspace: - # Building response from MutinyNet API - # Parse the UTXO data - utxos = gettx_req.json() - # Find the UTXO with the given txid and vout - for utxo in utxos: - if utxo['txid'] == txid: - return { - "amount": utxo["value"], - "script": None # MutinyNet API does not provide script information - } - - # If the specific UTXO is not found - return { - "amount": None, - "script": None - } - else: - # Building response from Blockstream API - status_req = fetch(status_url) - if not status_req.status_code == 200: - raise SauronError( - "Endpoint at {} returned {} ({}) when trying to get UTXO status.".format( - status_url, status_req.status_code, status_req.text - ) + status_req = fetch(status_url) + if not status_req.status_code == 200: + raise SauronError( + "Endpoint at {} returned {} ({}) when trying to " "get utxo status.".format( + status_url, status_req.status_code, status_req.text ) + ) - if status_req.json()["spent"]: - return { - "amount": None, - "script": None - } - - txo = gettx_req.json()["vout"][vout] + if status_req.json()["spent"]: return { - "amount": txo["value"], - "script": txo["scriptpubkey"] + "amount": None, + "script": None, } + txo = gettx_req.json()["vout"][vout] + return { + "amount": txo["value"], + "script": txo["scriptpubkey"], + } @plugin.method("estimatefees") def estimatefees(plugin, **kwargs): - # Define the URL based on the selected API if plugin.is_mempoolspace: # MutinyNet API feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) @@ -250,29 +212,21 @@ def estimatefees(plugin, **kwargs): # Blockstream API feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) - plugin.log("estimatefees: plugin.api_endpoint = %s" % plugin.api_endpoint) - plugin.log("estimatefees: feerate_url = %s" % feerate_url) feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 feerates = feerate_req.json() - plugin.log("estimatefees: feerates = %s" % feerates) - - # Define the multiply factor for sat/vB to sat/kVB conversion - multiply_factor = 10**3 - - if plugin.sauron_network in ["test", "signet"]: - # Apply the fallback for test/signet networks + if plugin.sauron_network == "test" or plugin.sauron_network == "signet": + # FIXME: remove the hack if the test API is "fixed" feerate = feerates.get("144", 1) - slow = normal = urgent = very_urgent = int(feerate * multiply_factor) + slow = normal = urgent = very_urgent = int(feerate * 10**3) else: - # Adjust fee rates based on the specific API # It returns sat/vB, we want sat/kVB, so multiply everything by 10**3 - slow = int(feerates["144"] * multiply_factor) - normal = int(feerates["12"] * multiply_factor) - urgent = int(feerates["6"] * multiply_factor) - very_urgent = int(feerates["2"] * multiply_factor) + slow = int(feerates["144"] * 10**3) + normal = int(feerates["12"] * 10**3) + urgent = int(feerates["6"] * 10**3) + very_urgent = int(feerates["2"] * 10**3) - feerate_floor = int(feerates.get("1008", slow) * multiply_factor) + feerate_floor = int(feerates.get("1008", slow) * 10**3) feerates = [ {"blocks": 2, "feerate": very_urgent}, {"blocks": 6, "feerate": urgent}, @@ -280,7 +234,6 @@ def estimatefees(plugin, **kwargs): {"blocks": 144, "feerate": slow} ] - # Return the estimated fees return { "opening": normal, "mutual_close": normal, @@ -294,6 +247,7 @@ def estimatefees(plugin, **kwargs): "feerates": feerates } + plugin.add_option( "sauron-api-endpoint", "", From 98f5faebd109b748be3b91a1f47425329828ed36 Mon Sep 17 00:00:00 2001 From: sip21 Date: Sat, 28 Sep 2024 02:13:27 -0600 Subject: [PATCH 6/8] Add sauron tests --- .ci/test.py | 3 +- poncho | 2 +- sauron/sauron.py | 11 +- sauron/tests/test_sauron_esplora.py | 152 +++++++++++++++++ sauron/tests/test_sauron_esplora_tor_proxy.py | 45 ++++++ sauron/tests/test_sauron_mempoolspace.py | 153 ++++++++++++++++++ sauron/tests/util.py | 34 ++++ 7 files changed, 393 insertions(+), 7 deletions(-) create mode 100644 sauron/tests/test_sauron_esplora.py create mode 100644 sauron/tests/test_sauron_esplora_tor_proxy.py create mode 100644 sauron/tests/test_sauron_mempoolspace.py create mode 100644 sauron/tests/util.py diff --git a/.ci/test.py b/.ci/test.py index a4f5ee742..7239d4594 100644 --- a/.ci/test.py +++ b/.ci/test.py @@ -241,13 +241,14 @@ def run_one(p: Plugin, workflow: str) -> bool: logging.info(f"Virtualenv at {vpath}") + num_workers = 1 if p.name == "sauron" else 5 cmd = [ str(pytest_path), "-vvv", "--timeout=600", "--timeout-method=thread", "--color=yes", - "-n=5", + f"-n={num_workers}", ] logging.info(f"Running `{' '.join(cmd)}` in directory {p.path.resolve()}") diff --git a/poncho b/poncho index e7795d763..a94da96ff 160000 --- a/poncho +++ b/poncho @@ -1 +1 @@ -Subproject commit e7795d763168d81435e7430105a7ef4c6985c45a +Subproject commit a94da96ff257cd10edda74ac897fc15a4344a08e diff --git a/sauron/sauron.py b/sauron/sauron.py index 0ac6b9375..4165ef100 100755 --- a/sauron/sauron.py +++ b/sauron/sauron.py @@ -51,9 +51,9 @@ def init(plugin, options, **kwargs): # Esplora API feerate_url = "{}/fee-estimates".format(plugin.api_endpoint) feerate_req = fetch(feerate_url) - assert feerate_req.status_code == 200 + assert feerate_req.status_code == 200 and feerate_req.content != b'{}' plugin.is_mempoolspace = False - except AssertionError as e0: + except AssertionError: try: # MutinyNet API feerate_url = "{}/v1/fees/recommended".format(plugin.api_endpoint) @@ -72,7 +72,8 @@ def init(plugin, options, **kwargs): } plugin.log("Using proxy {} for requests".format(socks5_proxy)) - plugin.log("Sauron plugin initialized") + api = "mempool.space" if plugin.is_mempoolspace else "Esplora" + plugin.log(f"Sauron plugin initialized using {api} API") plugin.log(sauron_eye) @@ -215,7 +216,7 @@ def estimatefees(plugin, **kwargs): feerate_req = fetch(feerate_url) assert feerate_req.status_code == 200 feerates = feerate_req.json() - if plugin.sauron_network == "test" or plugin.sauron_network == "signet": + if plugin.sauron_network in ["test", "signet"]: # FIXME: remove the hack if the test API is "fixed" feerate = feerates.get("144", 1) slow = normal = urgent = very_urgent = int(feerate * 10**3) @@ -259,7 +260,7 @@ def estimatefees(plugin, **kwargs): "", "Tor's SocksPort address in the form address:port, don't specify the" " protocol. If you didn't modify your torrc you want to put" - "'localhost:9050' here.", + " 'localhost:9050' here.", ) diff --git a/sauron/tests/test_sauron_esplora.py b/sauron/tests/test_sauron_esplora.py new file mode 100644 index 000000000..233f00965 --- /dev/null +++ b/sauron/tests/test_sauron_esplora.py @@ -0,0 +1,152 @@ +#!/usr/bin/python + +import os + +os.environ["TEST_NETWORK"] = "bitcoin" +import pyln +import pytest +from pyln.testing import utils +from util import * # noqa: F403 + +pyln.testing.fixtures.network_daemons["bitcoin"] = utils.BitcoinD + + +class LightningNode(utils.LightningNode): + def __init__(self, *args, **kwargs): + utils.LightningNode.__init__(self, *args, **kwargs) + lightning_dir = args[1] + + self.daemon = LightningD(lightning_dir, None) # noqa: F405 + options = { + "disable-plugin": "bcli", + "network": "bitcoin", + "plugin": os.path.join(os.path.dirname(__file__), "../sauron.py"), + "sauron-api-endpoint": "https://blockstream.info/api", + } + self.daemon.opts.update(options) + + # Monkey patch + def set_feerates(self, feerates, wait_for_effect=True): + return None + + +@pytest.fixture +def node_cls(): + yield LightningNode + + +def test_rpc_getchaininfo(ln_node): + """ + Test getchaininfo + """ + + response = ln_node.rpc.call("getchaininfo") + + assert ln_node.daemon.is_in_log("Sauron plugin initialized using Esplora API") + + expected_response_keys = ["chain", "blockcount", "headercount", "ibd"] + assert list(response.keys()) == expected_response_keys + assert response["chain"] == "main" + assert not response["ibd"] + + +def test_rpc_getrawblockbyheight(ln_node): + """ + Test getrawblockbyheight + """ + + response = ln_node.rpc.call("getrawblockbyheight", {"height": 0}) + + expected_response = { + "block": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a29ab5f49ffff001d1dac2b7c0101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", + "blockhash": "000000000019d6689c085ae165831e934ff763ae46a2a6c172b3f1b60a8ce26f", + } + assert response == expected_response + + +def test_rpc_sendrawtransaction_invalid(ln_node): + """ + Test sendrawtransaction + """ + + expected_response = { + "errmsg": 'sendrawtransaction RPC error: {"code":-22,"message":"TX decode failed. Make sure the tx has at least one input."}', + "success": False, + } + response = ln_node.rpc.call( + "sendrawtransaction", + {"tx": "invalid-raw-tx"}, + ) + + assert response == expected_response + + +def test_rpc_getutxout(ln_node): + """ + Test getutxout + """ + + expected_response = { + "amount": 1000000000, + "script": "4104b5abd412d4341b45056d3e376cd446eca43fa871b51961330deebd84423e740daa520690e1d9e074654c59ff87b408db903649623e86f1ca5412786f61ade2bfac", + } + response = ln_node.rpc.call( + "getutxout", + { + # block 181 + "txid": "a16f3ce4dd5deb92d98ef5cf8afeaf0775ebca408f708b2146c4fb42b41e14be", + "vout": 0, + }, + ) + assert response == expected_response + + +def test_rpc_estimatefees(ln_node): + """ + Test estimatefees + """ + + # Sample response + # { + # "opening": 4477, + # "mutual_close": 4477, + # "unilateral_close": 11929, + # "delayed_to_us": 4477, + # "htlc_resolution": 5652, + # "penalty": 5652, + # "min_acceptable": 1060, + # "max_acceptable": 119290, + # "feerate_floor": 1520, + # "feerates": [ + # {"blocks": 2, "feerate": 11929}, + # {"blocks": 6, "feerate": 5652}, + # {"blocks": 12, "feerate": 4477}, + # {"blocks": 144, "feerate": 2120} + # ] + # } + response = ln_node.rpc.call("estimatefees") + + expected_response_keys = [ + "opening", + "mutual_close", + "unilateral_close", + "delayed_to_us", + "htlc_resolution", + "penalty", + "min_acceptable", + "max_acceptable", + "feerate_floor", + "feerates", + ] + assert list(response.keys()) == expected_response_keys + + expected_feerates_keys = ("blocks", "feerate") + assert ( + list(set([tuple(entry.keys()) for entry in response["feerates"]]))[0] + == expected_feerates_keys + ) + + expected_feerates_blocks = [2, 6, 12, 144] + assert [ + entry["blocks"] for entry in response["feerates"] + ] == expected_feerates_blocks diff --git a/sauron/tests/test_sauron_esplora_tor_proxy.py b/sauron/tests/test_sauron_esplora_tor_proxy.py new file mode 100644 index 000000000..434335a9c --- /dev/null +++ b/sauron/tests/test_sauron_esplora_tor_proxy.py @@ -0,0 +1,45 @@ +#!/usr/bin/python + +import os + +os.environ["TEST_NETWORK"] = "bitcoin" +import pyln +import pytest +from pyln.testing import utils +from util import * # noqa: F403 + +pyln.testing.fixtures.network_daemons["bitcoin"] = utils.BitcoinD + + +class LightningNode(utils.LightningNode): + def __init__(self, *args, **kwargs): + utils.LightningNode.__init__(self, *args, **kwargs) + lightning_dir = args[1] + + self.daemon = LightningD(lightning_dir, None) # noqa: F405 + options = { + "disable-plugin": "bcli", + "network": "bitcoin", + "plugin": os.path.join(os.path.dirname(__file__), "../sauron.py"), + "sauron-api-endpoint": "https://blockstream.info/api", + "sauron-tor-proxy": "localhost:9050", + } + self.daemon.opts.update(options) + + # Monkey patch + def set_feerates(self, feerates, wait_for_effect=True): + return None + + +@pytest.fixture +def node_cls(): + yield LightningNode + + +def test_tor_proxy(ln_node): + """ + Test for tor proxy + """ + + assert ln_node.daemon.opts["sauron-tor-proxy"] == "localhost:9050" + assert ln_node.daemon.is_in_log("Using proxy socks5h://localhost:9050 for requests") diff --git a/sauron/tests/test_sauron_mempoolspace.py b/sauron/tests/test_sauron_mempoolspace.py new file mode 100644 index 000000000..c07b970b4 --- /dev/null +++ b/sauron/tests/test_sauron_mempoolspace.py @@ -0,0 +1,153 @@ +#!/usr/bin/python + +import os + +os.environ["TEST_NETWORK"] = "signet" +import pyln +import pytest +from pyln.testing import utils +from util import * # noqa: F403 + +pyln.testing.fixtures.network_daemons["signet"] = utils.BitcoinD + + +class LightningNode(utils.LightningNode): + def __init__(self, *args, **kwargs): + utils.LightningNode.__init__(self, *args, **kwargs) + lightning_dir = args[1] + + self.daemon = LightningD(lightning_dir, None) # noqa: F405 + options = { + "disable-plugin": "bcli", + "network": "signet", + "plugin": os.path.join(os.path.dirname(__file__), "../sauron.py"), + "sauron-api-endpoint": "https://mutinynet.com/api", + } + self.daemon.opts.update(options) + + # Monkey patch + def set_feerates(self, feerates, wait_for_effect=True): + return None + + +@pytest.fixture +def node_cls(): + yield LightningNode + + +def test_rpc_getchaininfo(ln_node): + """ + Test getchaininfo + """ + + response = ln_node.rpc.call("getchaininfo") + + assert ln_node.daemon.is_in_log("Sauron plugin initialized using mempool.space API") + + expected_response_keys = ["chain", "blockcount", "headercount", "ibd"] + assert list(response.keys()) == expected_response_keys + assert response["chain"] == "signet" + assert not response["ibd"] + + +def test_rpc_getrawblockbyheight(ln_node): + """ + Test getrawblockbyheight + """ + + response = ln_node.rpc.call("getrawblockbyheight", {"height": 0}) + + expected_response = { + "block": "0100000000000000000000000000000000000000000000000000000000000000000000003ba3edfd7a7b12b27ac72c3e67768f617fc81bc3888a51323a9fb8aa4b1e5e4a008f4d5fae77031e8ad222030101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff4d04ffff001d0104455468652054696d65732030332f4a616e2f32303039204368616e63656c6c6f72206f6e206272696e6b206f66207365636f6e64206261696c6f757420666f722062616e6b73ffffffff0100f2052a01000000434104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac00000000", + "blockhash": "00000008819873e925422c1ff0f99f7cc9bbb232af63a077a480a3633bee1ef6", + } + assert response == expected_response + + +def test_rpc_sendrawtransaction_invalid(ln_node): + """ + Test sendrawtransaction + """ + + expected_response = { + "errmsg": 'sendrawtransaction RPC error: {"code":-22,"message":"TX decode failed. Make sure the tx has at least one input."}', + "success": False, + } + response = ln_node.rpc.call( + "sendrawtransaction", + {"tx": "invalid-raw-tx"}, + ) + + assert response == expected_response + + +def test_rpc_getutxout(ln_node): + """ + Test getutxout + """ + + expected_response = { + "amount": 5000000000, + "script": "4104678afdb0fe5548271967f1a67130b7105cd6a828e03909a67962e0ea1f61deb649f6bc3f4cef38c4f35504e51ec112de5c384df7ba0b8d578a4c702b6bf11d5fac", + } + response = ln_node.rpc.call( + "getutxout", + { + # coinbase tx block 0 + "txid": "4a5e1e4baab89f3a32518a88c31bc87f618f76673e2cc77ab2127b7afdeda33b", + "vout": 0, + }, + ) + assert response == expected_response + + +def test_rpc_estimatefees(ln_node): + """ + Test estimatefees + """ + + # Sample response + # { + # "opening": 1000, + # "mutual_close": 1000, + # "unilateral_close": 1000, + # "delayed_to_us": 1000, + # "htlc_resolution": 1000, + # "penalty": 1000, + # "min_acceptable": 500, + # "max_acceptable": 10000, + # "feerate_floor": 1000000, + # "feerates": [ + # {"blocks": 2, "feerate": 1000}, + # {"blocks": 6, "feerate": 1000}, + # {"blocks": 12, "feerate": 1000}, + # {"blocks": 144, "feerate": 1000} + # ] + # } + + response = ln_node.rpc.call("estimatefees") + + expected_response_keys = [ + "opening", + "mutual_close", + "unilateral_close", + "delayed_to_us", + "htlc_resolution", + "penalty", + "min_acceptable", + "max_acceptable", + "feerate_floor", + "feerates", + ] + assert list(response.keys()) == expected_response_keys + + expected_feerates_keys = ("blocks", "feerate") + assert ( + list(set([tuple(entry.keys()) for entry in response["feerates"]]))[0] + == expected_feerates_keys + ) + + expected_feerates_blocks = [2, 6, 12, 144] + assert [ + entry["blocks"] for entry in response["feerates"] + ] == expected_feerates_blocks diff --git a/sauron/tests/util.py b/sauron/tests/util.py new file mode 100644 index 000000000..c0e01d2d1 --- /dev/null +++ b/sauron/tests/util.py @@ -0,0 +1,34 @@ +import logging + +import pytest +from pyln.testing import utils +from pyln.testing.fixtures import * # noqa: F403 + + +class LightningD(utils.LightningD): + def __init__(self, lightning_dir, *args, **kwargs): + super().__init__(lightning_dir, *args, **kwargs) + + opts_to_disable = [ + "bitcoin-datadir", + "bitcoin-rpcpassword", + "bitcoin-rpcuser", + "dev-bitcoind-poll", + ] + for opt in opts_to_disable: + self.opts.pop(opt) + + # Monkey patch + def start(self, stdin=None, wait_for_initialized=True, stderr_redir=False): + utils.TailableProc.start( + self, stdin, stdout_redir=False, stderr_redir=stderr_redir + ) + + if wait_for_initialized: + self.wait_for_log("Server started with public key") + logging.info("LightningD started") + + +@pytest.fixture +def ln_node(node_factory): # noqa: F811 + yield node_factory.get_node() From 21a42b4f85beacf64b6e4df7fd0c2c7b3a7c0b97 Mon Sep 17 00:00:00 2001 From: sip21 Date: Wed, 9 Oct 2024 13:44:11 -0600 Subject: [PATCH 7/8] Fix TEST_NETWORK --- sauron/tests/test_sauron_esplora.py | 5 +++-- sauron/tests/test_sauron_esplora_tor_proxy.py | 5 +++-- sauron/tests/test_sauron_mempoolspace.py | 5 +++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/sauron/tests/test_sauron_esplora.py b/sauron/tests/test_sauron_esplora.py index 233f00965..952fb7feb 100644 --- a/sauron/tests/test_sauron_esplora.py +++ b/sauron/tests/test_sauron_esplora.py @@ -2,7 +2,6 @@ import os -os.environ["TEST_NETWORK"] = "bitcoin" import pyln import pytest from pyln.testing import utils @@ -13,6 +12,7 @@ class LightningNode(utils.LightningNode): def __init__(self, *args, **kwargs): + pyln.testing.utils.TEST_NETWORK = "bitcoin" utils.LightningNode.__init__(self, *args, **kwargs) lightning_dir = args[1] @@ -31,7 +31,8 @@ def set_feerates(self, feerates, wait_for_effect=True): @pytest.fixture -def node_cls(): +def node_cls(monkeypatch): + monkeypatch.setenv("TEST_NETWORK", "bitcoin") yield LightningNode diff --git a/sauron/tests/test_sauron_esplora_tor_proxy.py b/sauron/tests/test_sauron_esplora_tor_proxy.py index 434335a9c..87783abe7 100644 --- a/sauron/tests/test_sauron_esplora_tor_proxy.py +++ b/sauron/tests/test_sauron_esplora_tor_proxy.py @@ -2,7 +2,6 @@ import os -os.environ["TEST_NETWORK"] = "bitcoin" import pyln import pytest from pyln.testing import utils @@ -13,6 +12,7 @@ class LightningNode(utils.LightningNode): def __init__(self, *args, **kwargs): + pyln.testing.utils.TEST_NETWORK = "bitcoin" utils.LightningNode.__init__(self, *args, **kwargs) lightning_dir = args[1] @@ -32,7 +32,8 @@ def set_feerates(self, feerates, wait_for_effect=True): @pytest.fixture -def node_cls(): +def node_cls(monkeypatch): + monkeypatch.setenv("TEST_NETWORK", "bitcoin") yield LightningNode diff --git a/sauron/tests/test_sauron_mempoolspace.py b/sauron/tests/test_sauron_mempoolspace.py index c07b970b4..c3e93b287 100644 --- a/sauron/tests/test_sauron_mempoolspace.py +++ b/sauron/tests/test_sauron_mempoolspace.py @@ -2,7 +2,6 @@ import os -os.environ["TEST_NETWORK"] = "signet" import pyln import pytest from pyln.testing import utils @@ -13,6 +12,7 @@ class LightningNode(utils.LightningNode): def __init__(self, *args, **kwargs): + pyln.testing.utils.TEST_NETWORK = "signet" utils.LightningNode.__init__(self, *args, **kwargs) lightning_dir = args[1] @@ -31,7 +31,8 @@ def set_feerates(self, feerates, wait_for_effect=True): @pytest.fixture -def node_cls(): +def node_cls(monkeypatch): + monkeypatch.setenv("TEST_NETWORK", "signet") yield LightningNode From e0e660398f3daae0d3a282e10a01d4075a2a642f Mon Sep 17 00:00:00 2001 From: sip21 Date: Wed, 9 Oct 2024 13:55:40 -0600 Subject: [PATCH 8/8] Fix TEST_NETWORK --- sauron/tests/test_sauron_esplora_tor_proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sauron/tests/test_sauron_esplora_tor_proxy.py b/sauron/tests/test_sauron_esplora_tor_proxy.py index 87783abe7..1ffd41d2d 100644 --- a/sauron/tests/test_sauron_esplora_tor_proxy.py +++ b/sauron/tests/test_sauron_esplora_tor_proxy.py @@ -36,7 +36,7 @@ def node_cls(monkeypatch): monkeypatch.setenv("TEST_NETWORK", "bitcoin") yield LightningNode - +@pytest.mark.skip(reason="TODO: Mock tor") def test_tor_proxy(ln_node): """ Test for tor proxy