From 832ab7d346e5888320b8ba98fbf4cac922d55bf6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 16:29:41 -0700 Subject: [PATCH 1/6] Start constructor migration --- README.md | 7 +- .../config.json} | 0 .../bitcoin.ros} | 197 ++++++++---------- rosetta-cli-conf/testnet/config.json | 50 +++++ 4 files changed, 138 insertions(+), 116 deletions(-) rename rosetta-cli-conf/{bitcoin_mainnet.json => mainnet/config.json} (100%) rename rosetta-cli-conf/{bitcoin_testnet.json => testnet/bitcoin.ros} (78%) create mode 100644 rosetta-cli-conf/testnet/config.json diff --git a/README.md b/README.md index 5865e97..3c71dfe 100644 --- a/README.md +++ b/README.md @@ -207,10 +207,9 @@ in recently processed blocks to save to disk. ## Testing with rosetta-cli To validate `rosetta-bitcoin`, [install `rosetta-cli`](https://github.com/coinbase/rosetta-cli#install) and run one of the following commands: -* `rosetta-cli check:data --configuration-file rosetta-cli-conf/bitcoin_testnet.json` -* `rosetta-cli check:construction --configuration-file rosetta-cli-conf/bitcoin_testnet.json` -* `rosetta-cli check:data --configuration-file rosetta-cli-conf/bitcoin_mainnet.json` -* `rosetta-cli check:construction --configuration-file rosetta-cli-conf/bitcoin_mainnet.json` +* `rosetta-cli check:data --configuration-file rosetta-cli-conf/testnet/config.json` +* `rosetta-cli check:construction --configuration-file rosetta-cli-conf/testnet/config.json` +* `rosetta-cli check:data --configuration-file rosetta-cli-conf/mainnet/config.json` ## Future Work * Publish benchamrks for sync speed, storage usage, and load testing diff --git a/rosetta-cli-conf/bitcoin_mainnet.json b/rosetta-cli-conf/mainnet/config.json similarity index 100% rename from rosetta-cli-conf/bitcoin_mainnet.json rename to rosetta-cli-conf/mainnet/config.json diff --git a/rosetta-cli-conf/bitcoin_testnet.json b/rosetta-cli-conf/testnet/bitcoin.ros similarity index 78% rename from rosetta-cli-conf/bitcoin_testnet.json rename to rosetta-cli-conf/testnet/bitcoin.ros index 90ceee5..e9f3aa2 100644 --- a/rosetta-cli-conf/bitcoin_testnet.json +++ b/rosetta-cli-conf/testnet/bitcoin.ros @@ -1,87 +1,88 @@ -{ - "network": { - "blockchain": "Bitcoin", - "network": "Testnet3" - }, - "data_directory": "bitcoin-testnet-data", - "http_timeout": 300, - "max_retries": 5, - "retry_elapsed_time": 0, - "max_online_connections": 0, - "max_sync_concurrency": 0, - "tip_delay": 1800, - "log_configuration": false, - "construction": { - "max_offline_connections": 0, - "stale_depth": 0, - "broadcast_limit": 0, - "ignore_broadcast_failures": false, - "clear_broadcasts": false, - "broadcast_behind_tip": false, - "block_broadcast_limit": 0, - "rebroadcast_all": false, - "workflows": [ - { - "name": "request_funds", - "concurrency": 1, - "scenarios": [ - { - "name": "find_account", - "actions": [ - { - "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", - "type": "find_balance", - "output_path": "random_account" - } - ] - }, - { - "name": "request", - "actions": [ - { - "input": "{\"account_identifier\": {{random_account.account_identifier}}, \"minimum_balance\":{\"value\": \"1000000\", \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "loaded_account" - } - ] - } - ] - }, - { - "name": "create_account", - "concurrency": 1, - "scenarios": [ - { - "name": "create_account", - "actions": [ - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "network" - }, - { - "input": "{\"curve_type\": \"secp256k1\"}", - "type": "generate_key", - "output_path": "key" - }, - { - "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", - "type": "derive", - "output_path": "account" - }, - { - "input": "{\"account_identifier\": {{account.account_identifier}}, \"keypair\": {{key}}}", - "type": "save_account" - } - ] - } - ] - }, +request_funds(1) { + find_account{ + currency = {"symbol":"tBTC", "decimals":8}; + random_account = find_balance({ + "minimum_balance":{ + "value": "0", + "currency": {{currency}} + }, + "create_limit":1 + }); + }, + + // Create a separate scenario to request funds so that + // the address we are using to request funds does not + // get rolled back if funds do not yet exist. + request{ + loaded_account = find_balance({ + "account_identifier": {{random_account.account_identifier}}, + "minimum_balance":{ + "value": "1000000", + "currency": {{currency}} + } + }); + } +} + +create_account(1){ + create{ + network = {"network":"Testnet3", "blockchain":"Bitcoin"}; + key = generate_key({"curve_type": "secp256k1"}); + account = derive({ + "network_identifier": {{network}}, + "public_key": {{key.public_key}} + }); + + // If the account is not saved, the key will be lost! + save_account({ + "account_identifier": {{account.account_identifier}}, + "keypair": {{key}} + }); + } +} + +transfer(10){ + transfer_dry_run{ + transfer_dry_run.network = {"network":"Testnet3", "blockchain":"Bitcoin"}; + currency = {"symbol":"tBTC", "decimals":8}; + + // We set the max_fee_amount to know how much buffer we should + // leave for fee payment when selecting a sender account. + dust_amount = "600"; + max_fee_amount = "1200"; + send_buffer = {{dust_amount}} + {{max_fee_amount}}; + + // reserved_amount is max_fee_amount + min_utxo size x 2 + reserved_amount = "2400"; + sender = find_balance({ + "minimum_balance":{ + "value": {{reserved_amount}}, + "currency": {{currency}} + }, + "require_coin": true + }); + + available_amount = {{sender.balance.value}} - {{send_buffer}}; + recipient_amount = random_number({ + "minimum": {{dust_amount}}, + "maximum": {{available_amount}} + }); + print_message({ + "recipient_amount":{{recipient_amount}} + }); + + total_change_amount = {{sender.balance.value}} - {{recipient_amount}}; + change_amount = {{total_change_amount}} - {{max_fee_amount}}; + print_message({ + "change_amount":{{change_amount}} + }); + + sender_amount = 0 - {{sender.balance.value}}; + + }, + transfer{ + }, +} { "name": "transfer", "concurrency": 10, @@ -350,31 +351,3 @@ } ] } - ], - "end_conditions": { - "create_account": 10, - "transfer": 10 - } - }, - "data": { - "active_reconciliation_concurrency": 0, - "inactive_reconciliation_concurrency": 0, - "inactive_reconciliation_frequency": 0, - "log_blocks": false, - "log_transactions": false, - "log_balance_changes": false, - "log_reconciliations": false, - "ignore_reconciliation_error": false, - "exempt_accounts": "", - "bootstrap_balances": "", - "interesting_accounts": "", - "reconciliation_disabled": false, - "inactive_discrepency_search_disabled": false, - "balance_tracking_disabled": false, - "coin_tracking_disabled": false, - "end_conditions": { - "reconciliation_coverage": 0.95 - }, - "results_output_file": "" - } -} diff --git a/rosetta-cli-conf/testnet/config.json b/rosetta-cli-conf/testnet/config.json new file mode 100644 index 0000000..c73810f --- /dev/null +++ b/rosetta-cli-conf/testnet/config.json @@ -0,0 +1,50 @@ +{ + "network": { + "blockchain": "Bitcoin", + "network": "Testnet3" + }, + "data_directory": "bitcoin-testnet-data", + "http_timeout": 300, + "max_retries": 5, + "retry_elapsed_time": 0, + "max_online_connections": 0, + "max_sync_concurrency": 0, + "tip_delay": 1800, + "log_configuration": false, + "construction": { + "max_offline_connections": 0, + "stale_depth": 0, + "broadcast_limit": 0, + "ignore_broadcast_failures": false, + "clear_broadcasts": false, + "broadcast_behind_tip": false, + "block_broadcast_limit": 0, + "rebroadcast_all": false, + "constructor_dsl_file": "bitcoin.ros", + "end_conditions": { + "create_account": 10, + "transfer": 10 + } + }, + "data": { + "active_reconciliation_concurrency": 0, + "inactive_reconciliation_concurrency": 0, + "inactive_reconciliation_frequency": 0, + "log_blocks": false, + "log_transactions": false, + "log_balance_changes": false, + "log_reconciliations": false, + "ignore_reconciliation_error": false, + "exempt_accounts": "", + "bootstrap_balances": "", + "interesting_accounts": "", + "reconciliation_disabled": false, + "inactive_discrepency_search_disabled": false, + "balance_tracking_disabled": false, + "coin_tracking_disabled": false, + "end_conditions": { + "reconciliation_coverage": 0.95 + }, + "results_output_file": "" + } +} From c5819a55b5f7b0fa02f04311aa5d7c683c402f60 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 17:14:16 -0700 Subject: [PATCH 2/6] First pass at bitcoin.ros --- rosetta-cli-conf/testnet/bitcoin.ros | 402 +++++++++------------------ 1 file changed, 133 insertions(+), 269 deletions(-) diff --git a/rosetta-cli-conf/testnet/bitcoin.ros b/rosetta-cli-conf/testnet/bitcoin.ros index e9f3aa2..4092080 100644 --- a/rosetta-cli-conf/testnet/bitcoin.ros +++ b/rosetta-cli-conf/testnet/bitcoin.ros @@ -78,276 +78,140 @@ transfer(10){ }); sender_amount = 0 - {{sender.balance.value}}; - + recipient = find_balance({ + "not_account_identifier":[{{sender.account_identifier}}], + "not_coins":[{{sender.coin}}], + "minimum_balance":{ + "value": "0", + "currency": {{currency}} + }, + "create_limit": 100, + "create_probability": 50 + }); + transfer_dry_run.confirmation_depth = "1"; + transfer_dry_run.dry_run = true; + transfer_dry_run.operations = [ + { + "operation_identifier":{"index":0}, + "type":"INPUT", + "account":{{sender.account_identifier}}, + "amount":{"value":{{sender_amount}},"currency":{{currency}}}, + "coin_change":{"coin_action":"coin_spent", "coin_identifier":{{sender.coin}}} + }, + { + "operation_identifier":{"index":1}, + "type":"OUTPUT", + "account":{{recipient.account_identifier}}, + "amount":{"value":{{recipient_amount}},"currency":{{currency}}} + }, + { + "operation_identifier":{"index":2}, + "type":"OUTPUT", + "account":{{sender.account_identifier}}, + "amount":{"value":{{change_amount}},"currency":{{currency}}} + } + ]; }, transfer{ + suggested_fee = find_currency_amount({ + "currency":{{currency}}, + "amounts":{{transfer_dry_run.suggested_fee}} + }); + change_amount = {{total_change_amount}} - {{suggested_fee.value}}; + change_minus_dust = {{change_amount}} - {{dust_amount}}; + transfer.network = {{transfer_dry_run.network}}; + transfer.confirmation_depth = {{transfer_dry_run.confirmation_depth}}; + transfer.operations = [ + { + "operation_identifier":{"index":0}, + "type":"INPUT", + "account":{{sender.account_identifier}}, + "amount":{"value":{{sender_amount}},"currency":{{currency}}}, + "coin_change":{"coin_action":"coin_spent", "coin_identifier":{{sender.coin}}} + }, + { + "operation_identifier":{"index":1}, + "type":"OUTPUT", + "account":{{recipient.account_identifier}}, + "amount":{"value":{{recipient_amount}},"currency":{{currency}}} + }, + { + "operation_identifier":{"index":2}, + "type":"OUTPUT", + "account":{{sender.account_identifier}}, + "amount":{"value":{{change_amount}},"currency":{{currency}}} + } + ]; + }, +} + +return_funds(10){ + transfer_dry_run{ + transfer_dry_run.network = {"network":"Testnet3", "blockchain":"Bitcoin"}; + currency = {"symbol":"tBTC", "decimals":8}; + + max_fee_amount = "1200"; + + // reserved_amount is max_fee_amount + min_utxo size + reserved_amount = "1800"; + sender = find_balance({ + "minimum_balance":{ + "value": {{reserved_amount}}, + "currency": {{currency}} + }, + "require_coin": true + }); + + recipient_amount = {{sender.balance.value}} - {{max_fee_amount}}; + print_message({ + "recipient_amount":{{recipient_amount}} + }); + + sender_amount = 0 - {{sender.balance.value}}; + recipient = {"address": "mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt"}; + transfer_dry_run.confirmation_depth = "1"; + transfer_dry_run.dry_run = true; + transfer_dry_run.operations = [ + { + "operation_identifier":{"index":0}, + "type":"INPUT", + "account":{{sender.account_identifier}}, + "amount":{"value":{{sender_amount}},"currency":{{currency}}}, + "coin_change":{"coin_action":"coin_spent", "coin_identifier":{{sender.coin}}} + }, + { + "operation_identifier":{"index":1}, + "type":"OUTPUT", + "account":{{recipient}}, + "amount":{"value":{{recipient_amount}},"currency":{{currency}}} + } + ]; + }, + transfer{ + suggested_fee = find_currency_amount({ + "currency":{{currency}}, + "amounts":{{transfer_dry_run.suggested_fee}} + }); + recipient_amount = {{sender.balance.value}} - {{suggested_fee.value}}; + dust_amount = "600"; + recipient_minus_dust = {{recipient_amount}} - {{dust_amount}}; + assert({{recipient_minus_dust}}); + transfer.network = {{transfer_dry_run.network}}; + transfer.confirmation_depth = {{transfer_dry_run.confirmation_depth}}; + transfer.operations = [ + { + "operation_identifier":{"index":0}, + "type":"INPUT", + "account":{{sender.account_identifier}}, + "amount":{"value":{{sender_amount}},"currency":{{currency}}}, + "coin_change":{"coin_action":"coin_spent", "coin_identifier":{{sender.coin}}} + }, + { + "operation_identifier":{"index":1}, + "type":"OUTPUT", + "account":{{recipient}}, + "amount":{"value":{{recipient_amount}},"currency":{{currency}}} + } + ]; }, } - { - "name": "transfer", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer_dry_run", - "actions": [ - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer_dry_run.network" - }, - { - "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"600\"", - "type": "set_variable", - "output_path": "dust_amount" - }, - { - "input": "\"1200\"", - "type": "set_variable", - "output_path": "max_fee_amount" - }, - { - "input": "{\"operation\":\"addition\", \"left_value\": {{dust_amount}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "send_buffer" - }, - { - "input": "\"2400\"", - "type": "set_variable", - "output_path": "reserved_amount" - }, - { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{send_buffer}}}", - "type": "math", - "output_path": "available_amount" - }, - { - "input": "{\"minimum\": {{dust_amount}}, \"maximum\": {{available_amount}}}", - "type": "random_number", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{recipient_amount}}}", - "type": "math", - "output_path": "total_change_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "change_amount" - }, - { - "input": "{\"change_amount\":{{change_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "{\"not_account_identifier\":[{{sender.account_identifier}}], \"not_coins\":[{{sender.coin}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", - "type": "find_balance", - "output_path": "recipient" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer_dry_run.confirmation_depth" - }, - { - "input": "\"true\"", - "type": "set_variable", - "output_path": "transfer_dry_run.dry_run" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer_dry_run.operations" - }, - { - "input": "{{transfer_dry_run.operations}}", - "type": "print_message" - } - ] - }, - { - "name": "transfer", - "actions": [ - { - "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", - "type": "find_currency_amount", - "output_path": "suggested_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{suggested_fee.value}}}", - "type": "math", - "output_path": "change_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{change_amount}}, \"right_value\": {{dust_amount}}}", - "type": "math", - "output_path": "change_minus_dust" - }, - { - "input": "{{change_minus_dust}}", - "type": "assert" - }, - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - }, - { - "input": "{{transfer.operations}}", - "type": "print_message" - } - ] - } - ] - }, - { - "name": "return_funds", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer_dry_run", - "actions": [ - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer_dry_run.network" - }, - { - "input": "{\"symbol\":\"tBTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"1200\"", - "type": "set_variable", - "output_path": "max_fee_amount" - }, - { - "input": "\"1800\"", - "type": "set_variable", - "output_path": "reserved_amount" - }, - { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer_dry_run.confirmation_depth" - }, - { - "input": "\"true\"", - "type": "set_variable", - "output_path": "transfer_dry_run.dry_run" - }, - { - "input": "{\"address\": \"mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt\"}", - "type": "set_variable", - "output_path": "recipient" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer_dry_run.operations" - }, - { - "input": "{{transfer_dry_run.operations}}", - "type": "print_message" - } - ] - }, - { - "name": "transfer", - "actions": [ - { - "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", - "type": "find_currency_amount", - "output_path": "suggested_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{suggested_fee.value}}}", - "type": "math", - "output_path": "recipient_amount" - }, - { - "input": "\"600\"", - "type": "set_variable", - "output_path": "dust_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{recipient_amount}}, \"right_value\": {{dust_amount}}}", - "type": "math", - "output_path": "recipient_minus_dust" - }, - { - "input": "{{recipient_minus_dust}}", - "type": "assert" - }, - { - "input": "{\"network\":\"Testnet3\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - }, - { - "input": "{{transfer.operations}}", - "type": "print_message" - } - ] - } - ] - } From fd8cea864e339215832bb74463a6944302c49218 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 17:25:11 -0700 Subject: [PATCH 3/6] Delete construction testing for mainnet --- rosetta-cli-conf/mainnet/config.json | 347 +-------------------------- rosetta-cli-conf/testnet/config.json | 2 +- 2 files changed, 2 insertions(+), 347 deletions(-) diff --git a/rosetta-cli-conf/mainnet/config.json b/rosetta-cli-conf/mainnet/config.json index be17d0a..dfbbfce 100644 --- a/rosetta-cli-conf/mainnet/config.json +++ b/rosetta-cli-conf/mainnet/config.json @@ -3,7 +3,7 @@ "blockchain": "Bitcoin", "network": "Mainnet" }, - "data_directory": "bitcoin-mainnet-data", + "data_directory": "cli-data", "http_timeout": 300, "max_retries": 5, "retry_elapsed_time": 0, @@ -11,351 +11,6 @@ "max_sync_concurrency": 0, "tip_delay": 1800, "log_configuration": false, - "construction": { - "max_offline_connections": 0, - "stale_depth": 0, - "broadcast_limit": 0, - "ignore_broadcast_failures": false, - "clear_broadcasts": false, - "broadcast_behind_tip": false, - "block_broadcast_limit": 0, - "rebroadcast_all": false, - "workflows": [ - { - "name": "request_funds", - "concurrency": 1, - "scenarios": [ - { - "name": "find_account", - "actions": [ - { - "input": "{\"symbol\":\"BTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "{\"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\":1}", - "type": "find_balance", - "output_path": "random_account" - } - ] - }, - { - "name": "request", - "actions": [ - { - "input": "{\"account_identifier\": {{random_account.account_identifier}}, \"minimum_balance\":{\"value\": \"1000000\", \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "loaded_account" - } - ] - } - ] - }, - { - "name": "create_account", - "concurrency": 1, - "scenarios": [ - { - "name": "create_account", - "actions": [ - { - "input": "{\"network\":\"Mainnet\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "network" - }, - { - "input": "{\"curve_type\": \"secp256k1\"}", - "type": "generate_key", - "output_path": "key" - }, - { - "input": "{\"network_identifier\": {{network}}, \"public_key\": {{key.public_key}}}", - "type": "derive", - "output_path": "account" - }, - { - "input": "{\"account_identifier\": {{account.account_identifier}}, \"keypair\": {{key}}}", - "type": "save_account" - } - ] - } - ] - }, - { - "name": "transfer", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer_dry_run", - "actions": [ - { - "input": "{\"network\":\"Mainnet\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer_dry_run.network" - }, - { - "input": "{\"symbol\":\"BTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"600\"", - "type": "set_variable", - "output_path": "dust_amount" - }, - { - "input": "\"1200\"", - "type": "set_variable", - "output_path": "max_fee_amount" - }, - { - "input": "{\"operation\":\"addition\", \"left_value\": {{dust_amount}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "send_buffer" - }, - { - "input": "\"2400\"", - "type": "set_variable", - "output_path": "reserved_amount" - }, - { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{send_buffer}}}", - "type": "math", - "output_path": "available_amount" - }, - { - "input": "{\"minimum\": {{dust_amount}}, \"maximum\": {{available_amount}}}", - "type": "random_number", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{recipient_amount}}}", - "type": "math", - "output_path": "total_change_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "change_amount" - }, - { - "input": "{\"change_amount\":{{change_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "{\"not_account_identifier\":[{{sender.account_identifier}}], \"not_coins\":[{{sender.coin}}], \"minimum_balance\":{\"value\": \"0\", \"currency\": {{currency}}}, \"create_limit\": 100, \"create_probability\": 50}", - "type": "find_balance", - "output_path": "recipient" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer_dry_run.confirmation_depth" - }, - { - "input": "\"true\"", - "type": "set_variable", - "output_path": "transfer_dry_run.dry_run" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer_dry_run.operations" - }, - { - "input": "{{transfer_dry_run.operations}}", - "type": "print_message" - } - ] - }, - { - "name": "transfer", - "actions": [ - { - "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", - "type": "find_currency_amount", - "output_path": "suggested_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{total_change_amount}}, \"right_value\": {{suggested_fee.value}}}", - "type": "math", - "output_path": "change_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{change_amount}}, \"right_value\": {{dust_amount}}}", - "type": "math", - "output_path": "change_minus_dust" - }, - { - "input": "{{change_minus_dust}}", - "type": "assert" - }, - { - "input": "{\"network\":\"Mainnet\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient.account_identifier}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}, {\"operation_identifier\":{\"index\":2},\"type\":\"OUTPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{change_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - }, - { - "input": "{{transfer.operations}}", - "type": "print_message" - } - ] - } - ] - }, - { - "name": "return_funds", - "concurrency": 10, - "scenarios": [ - { - "name": "transfer_dry_run", - "actions": [ - { - "input": "{\"network\":\"Mainnet\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer_dry_run.network" - }, - { - "input": "{\"symbol\":\"BTC\", \"decimals\":8}", - "type": "set_variable", - "output_path": "currency" - }, - { - "input": "\"1200\"", - "type": "set_variable", - "output_path": "max_fee_amount" - }, - { - "input": "\"1800\"", - "type": "set_variable", - "output_path": "reserved_amount" - }, - { - "input": "{\"require_coin\":true, \"minimum_balance\":{\"value\": {{reserved_amount}}, \"currency\": {{currency}}}}", - "type": "find_balance", - "output_path": "sender" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{max_fee_amount}}}", - "type": "math", - "output_path": "recipient_amount" - }, - { - "input": "{\"recipient_amount\":{{recipient_amount}}}", - "type": "print_message" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": \"0\", \"right_value\":{{sender.balance.value}}}", - "type": "math", - "output_path": "sender_amount" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer_dry_run.confirmation_depth" - }, - { - "input": "\"true\"", - "type": "set_variable", - "output_path": "transfer_dry_run.dry_run" - }, - { - "input": "{\"address\": \"mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt\"}", - "type": "set_variable", - "output_path": "recipient" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer_dry_run.operations" - }, - { - "input": "{{transfer_dry_run.operations}}", - "type": "print_message" - } - ] - }, - { - "name": "transfer", - "actions": [ - { - "input": "{\"currency\":{{currency}}, \"amounts\":{{transfer_dry_run.suggested_fee}}}", - "type": "find_currency_amount", - "output_path": "suggested_fee" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{sender.balance.value}}, \"right_value\": {{suggested_fee.value}}}", - "type": "math", - "output_path": "recipient_amount" - }, - { - "input": "\"600\"", - "type": "set_variable", - "output_path": "dust_amount" - }, - { - "input": "{\"operation\":\"subtraction\", \"left_value\": {{recipient_amount}}, \"right_value\": {{dust_amount}}}", - "type": "math", - "output_path": "recipient_minus_dust" - }, - { - "input": "{{recipient_minus_dust}}", - "type": "assert" - }, - { - "input": "{\"network\":\"Mainnet\", \"blockchain\":\"Bitcoin\"}", - "type": "set_variable", - "output_path": "transfer.network" - }, - { - "input": "\"1\"", - "type": "set_variable", - "output_path": "transfer.confirmation_depth" - }, - { - "input": "[{\"operation_identifier\":{\"index\":0},\"type\":\"INPUT\",\"account\":{{sender.account_identifier}},\"amount\":{\"value\":{{sender_amount}},\"currency\":{{currency}}}, \"coin_change\":{\"coin_action\":\"coin_spent\", \"coin_identifier\":{{sender.coin}}}},{\"operation_identifier\":{\"index\":1},\"type\":\"OUTPUT\",\"account\":{{recipient}},\"amount\":{\"value\":{{recipient_amount}},\"currency\":{{currency}}}}]", - "type": "set_variable", - "output_path": "transfer.operations" - }, - { - "input": "{{transfer.operations}}", - "type": "print_message" - } - ] - } - ] - } - ], - "end_conditions": { - "create_account": 10, - "transfer": 10 - } - }, "data": { "active_reconciliation_concurrency": 0, "inactive_reconciliation_concurrency": 0, diff --git a/rosetta-cli-conf/testnet/config.json b/rosetta-cli-conf/testnet/config.json index c73810f..e0e649c 100644 --- a/rosetta-cli-conf/testnet/config.json +++ b/rosetta-cli-conf/testnet/config.json @@ -3,7 +3,7 @@ "blockchain": "Bitcoin", "network": "Testnet3" }, - "data_directory": "bitcoin-testnet-data", + "data_directory": "cli-data", "http_timeout": 300, "max_retries": 5, "retry_elapsed_time": 0, From c42c10cb32817e4bcacace7eed615f5698baa83c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 18:43:05 -0700 Subject: [PATCH 4/6] Tests passing --- .gitignore | 1 + rosetta-cli-conf/mainnet/config.json | 5 ++++- rosetta-cli-conf/testnet/bitcoin.ros | 6 +++--- rosetta-cli-conf/testnet/config.json | 5 ++++- 4 files changed, 12 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 2fab13c..5ded4fc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ rosetta-bitcoin bitcoin-data +cli-data diff --git a/rosetta-cli-conf/mainnet/config.json b/rosetta-cli-conf/mainnet/config.json index dfbbfce..2c0b8f5 100644 --- a/rosetta-cli-conf/mainnet/config.json +++ b/rosetta-cli-conf/mainnet/config.json @@ -28,7 +28,10 @@ "balance_tracking_disabled": false, "coin_tracking_disabled": false, "end_conditions": { - "reconciliation_coverage": 0.95 + "reconciliation_coverage": { + "coverage": 0.95, + "from_tip": true + } }, "results_output_file": "" } diff --git a/rosetta-cli-conf/testnet/bitcoin.ros b/rosetta-cli-conf/testnet/bitcoin.ros index 4092080..bae952c 100644 --- a/rosetta-cli-conf/testnet/bitcoin.ros +++ b/rosetta-cli-conf/testnet/bitcoin.ros @@ -1,4 +1,4 @@ -request_funds(1) { +request_funds(1){ find_account{ currency = {"symbol":"tBTC", "decimals":8}; random_account = find_balance({ @@ -142,7 +142,7 @@ transfer(10){ "amount":{"value":{{change_amount}},"currency":{{currency}}} } ]; - }, + } } return_funds(10){ @@ -213,5 +213,5 @@ return_funds(10){ "amount":{"value":{{recipient_amount}},"currency":{{currency}}} } ]; - }, + } } diff --git a/rosetta-cli-conf/testnet/config.json b/rosetta-cli-conf/testnet/config.json index e0e649c..7591bb5 100644 --- a/rosetta-cli-conf/testnet/config.json +++ b/rosetta-cli-conf/testnet/config.json @@ -43,7 +43,10 @@ "balance_tracking_disabled": false, "coin_tracking_disabled": false, "end_conditions": { - "reconciliation_coverage": 0.95 + "reconciliation_coverage": { + "coverage": 0.95, + "from_tip": true + } }, "results_output_file": "" } From dcdc1cf81f9ffda538a4998779184e560fcf3008 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 26 Oct 2020 18:59:35 -0700 Subject: [PATCH 5/6] Require coin in configuration --- rosetta-cli-conf/testnet/bitcoin.ros | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/rosetta-cli-conf/testnet/bitcoin.ros b/rosetta-cli-conf/testnet/bitcoin.ros index bae952c..a2c35c7 100644 --- a/rosetta-cli-conf/testnet/bitcoin.ros +++ b/rosetta-cli-conf/testnet/bitcoin.ros @@ -19,7 +19,8 @@ request_funds(1){ "minimum_balance":{ "value": "1000000", "currency": {{currency}} - } + }, + "require_coin":true }); } } From 68a2b1666e94ebb0978914d5d89ffbc4fa6395a8 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 27 Oct 2020 08:31:48 -0700 Subject: [PATCH 6/6] Add comments --- rosetta-cli-conf/testnet/bitcoin.ros | 50 ++++++++++++++++++++++------ 1 file changed, 39 insertions(+), 11 deletions(-) diff --git a/rosetta-cli-conf/testnet/bitcoin.ros b/rosetta-cli-conf/testnet/bitcoin.ros index a2c35c7..040e467 100644 --- a/rosetta-cli-conf/testnet/bitcoin.ros +++ b/rosetta-cli-conf/testnet/bitcoin.ros @@ -53,7 +53,8 @@ transfer(10){ max_fee_amount = "1200"; send_buffer = {{dust_amount}} + {{max_fee_amount}}; - // reserved_amount is max_fee_amount + min_utxo size x 2 + // We look for a coin of value >= the reserved_amount to create + // a transfer with change (reserved_amount is max_fee_amount + dust_amount x 2). reserved_amount = "2400"; sender = find_balance({ "minimum_balance":{ @@ -63,22 +64,32 @@ transfer(10){ "require_coin": true }); - available_amount = {{sender.balance.value}} - {{send_buffer}}; + // The amount we send to the recipient is a random value + // between the dust_amount and the value of the entire coin (minus + // the amount reserved for fee payment and covering the dust minimum + // of the change UTXO). + receivable_amount = {{sender.balance.value}} - {{send_buffer}}; recipient_amount = random_number({ "minimum": {{dust_amount}}, - "maximum": {{available_amount}} + "maximum": {{receivable_amount}} }); print_message({ "recipient_amount":{{recipient_amount}} }); - total_change_amount = {{sender.balance.value}} - {{recipient_amount}}; - change_amount = {{total_change_amount}} - {{max_fee_amount}}; + // The change amount is what we aren't sending to the recipient + // minus the maximum fee. Don't worry, we will adjust this + // amount to avoid overpaying the fee after the dry run + // completes. + raw_change_amount = {{sender.balance.value}} - {{recipient_amount}}; + change_amount = {{raw_change_amount}} - {{max_fee_amount}}; print_message({ "change_amount":{{change_amount}} }); - sender_amount = 0 - {{sender.balance.value}}; + // The last thing we need to do before creating the transaction + // is to find a recipient with a *types.AccountIdentifier that + // is not equal to the sender. recipient = find_balance({ "not_account_identifier":[{{sender.account_identifier}}], "not_coins":[{{sender.coin}}], @@ -89,6 +100,8 @@ transfer(10){ "create_limit": 100, "create_probability": 50 }); + + sender_amount = 0 - {{sender.balance.value}}; transfer_dry_run.confirmation_depth = "1"; transfer_dry_run.dry_run = true; transfer_dry_run.operations = [ @@ -114,12 +127,16 @@ transfer(10){ ]; }, transfer{ + // The suggested_fee is returned in the /construction/metadata + // response and saved to transfer_dry_run.suggested_fee. suggested_fee = find_currency_amount({ "currency":{{currency}}, "amounts":{{transfer_dry_run.suggested_fee}} }); - change_amount = {{total_change_amount}} - {{suggested_fee.value}}; - change_minus_dust = {{change_amount}} - {{dust_amount}}; + + // We can access the variables of other scenarios, so we don't + // need to recalculate raw_change_amount. + change_amount = {{raw_change_amount}} - {{suggested_fee.value}}; transfer.network = {{transfer_dry_run.network}}; transfer.confirmation_depth = {{transfer_dry_run.confirmation_depth}}; transfer.operations = [ @@ -151,9 +168,9 @@ return_funds(10){ transfer_dry_run.network = {"network":"Testnet3", "blockchain":"Bitcoin"}; currency = {"symbol":"tBTC", "decimals":8}; + // We look for a sender that is able to pay the + // max_fee_amount + min_utxo size (reserved_amount is max_fee_amount + min_utxo size). max_fee_amount = "1200"; - - // reserved_amount is max_fee_amount + min_utxo size reserved_amount = "1800"; sender = find_balance({ "minimum_balance":{ @@ -163,13 +180,18 @@ return_funds(10){ "require_coin": true }); + // We send the maximum amount available to the recipient. Don't worry + // we will modify this after the dry run to make sure we don't overpay. recipient_amount = {{sender.balance.value}} - {{max_fee_amount}}; print_message({ "recipient_amount":{{recipient_amount}} }); + // We load the recipient address from an ENV. + recipient_address = load_env("RECIPIENT"); + recipient = {"address": {{recipient_address}}}; + sender_amount = 0 - {{sender.balance.value}}; - recipient = {"address": "mkHS9ne12qx9pS9VojpwU5xtRd4T7X7ZUt"}; transfer_dry_run.confirmation_depth = "1"; transfer_dry_run.dry_run = true; transfer_dry_run.operations = [ @@ -189,14 +211,20 @@ return_funds(10){ ]; }, transfer{ + // The suggested_fee is returned in the /construction/metadata + // response and saved to transfer_dry_run.suggested_fee. suggested_fee = find_currency_amount({ "currency":{{currency}}, "amounts":{{transfer_dry_run.suggested_fee}} }); + + // We calculate the recipient_amount using the new suggested_fee + // and assert that it is above the minimum UTXO size. recipient_amount = {{sender.balance.value}} - {{suggested_fee.value}}; dust_amount = "600"; recipient_minus_dust = {{recipient_amount}} - {{dust_amount}}; assert({{recipient_minus_dust}}); + transfer.network = {{transfer_dry_run.network}}; transfer.confirmation_depth = {{transfer_dry_run.confirmation_depth}}; transfer.operations = [