Merge #12717: [REST] Handle UTXO retrieval when ignoring the mempool

9cb9af8 [REST] Handle UTXO retrieval when ignoring the mempool (Roman Zeyde)
1fdc7c4 Make CTxMemPool::isSpent() const (Roman Zeyde)

Pull request description:

  Current REST API always returns empty UTXO when invoked without `/checkmempool/` URL part.

  After the fix:
  ```
  $ curl -s http://localhost:8332/rest/getutxos/0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098-0.json | jq
  {
    "chainHeight": 514109,
    "chaintipHash": "0000000000000000001fe76d1445e8a6432fd2de04261dc9c5915311dc7ad6de",
    "bitmap": "1",
    "utxos": [
      {
        "height": 1,
        "value": 50,
        "scriptPubKey": {
          "asm": "0496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858ee OP_CHECKSIG",
          "hex": "410496b538e853519c726a2c91e61ec11600ae1390813a627c66fb8be7947be63c52da7589379515d4e0a604f8141781e62294721166bf621e73a82cbf2342c858eeac",
          "reqSigs": 1,
          "type": "pubkey",
          "addresses": [
            "12c6DSiU4Rq3P4ZxziKxzrL5LmMBrzjrJX"
          ]
        }
      }
    ]
  }
  ```

  Before the fix:
  ```
  $ curl -s http://localhost:8332/rest/getutxos/0e3e2357e806b6cdb1f70b54c3a3a17b6714ee1f0e68bebb44a74b1efd512098-0.json | jq
  {
    "chainHeight": 514109,
    "chaintipHash": "0000000000000000001fe76d1445e8a6432fd2de04261dc9c5915311dc7ad6de",
    "bitmap": "0",
    "utxos": []
  }
  ```

Tree-SHA512: 994a350cb34a3c8f5a7afbc169c6b177c5be6cf223b2071c62d63644819d416d3e10d1c58b244d9d351bae7233d2974aa5e9ebadd1b5d6218f5245558675be0d

src/rest.cpp

@@ -487,26 +487,28 @@ static bool rest_getutxos(HTTPRequest* req, const std::string& strURIPart)
     std::vector<bool> hits;
     bitmap.resize((vOutPoints.size() + 7) / 8);
     {
-        LOCK2(cs_main, mempool.cs);
+        auto process_utxos = [&vOutPoints, &outs, &hits](const CCoinsView& view, const CTxMemPool& mempool) {
-
+            for (const COutPoint& vOutPoint : vOutPoints) {
-        CCoinsView viewDummy;
+                Coin coin;
-        CCoinsViewCache view(&viewDummy);
+                bool hit = !mempool.isSpent(vOutPoint) && view.GetCoin(vOutPoint, coin);
-
+                hits.push_back(hit);
-        CCoinsViewCache& viewChain = *pcoinsTip;
+                if (hit) outs.emplace_back(std::move(coin));
-        CCoinsViewMemPool viewMempool(&viewChain, mempool);
-
-        if (fCheckMemPool)
-            view.SetBackend(viewMempool); // switch cache backend to db+mempool in case user likes to query mempool
-
-        for (size_t i = 0; i < vOutPoints.size(); i++) {
-            bool hit = false;
-            Coin coin;
-            if (view.GetCoin(vOutPoints[i], coin) && !mempool.isSpent(vOutPoints[i])) {
-                hit = true;
-                outs.emplace_back(std::move(coin));
             }
+        };
 
-            hits.push_back(hit);
+        if (fCheckMemPool) {
+            // use db+mempool as cache backend in case user likes to query mempool
+            LOCK2(cs_main, mempool.cs);
+            CCoinsViewCache& viewChain = *pcoinsTip;
+            CCoinsViewMemPool viewMempool(&viewChain, mempool);
+            process_utxos(viewMempool, mempool);
+        } else {
+            LOCK(cs_main);  // no need to lock mempool!
+            process_utxos(*pcoinsTip, CTxMemPool());
+        }
+
+        for (size_t i = 0; i < hits.size(); ++i) {
+            const bool hit = hits[i];
             bitmapStringRepresentation.append(hit ? "1" : "0"); // form a binary string representation (human-readable for json output)
             bitmap[i / 8] |= ((uint8_t)hit) << (i % 8);
         }

src/txmempool.cpp

@@ -339,7 +339,7 @@ CTxMemPool::CTxMemPool(CBlockPolicyEstimator* estimator) :
     nCheckFrequency = 0;
 }
 
-bool CTxMemPool::isSpent(const COutPoint& outpoint)
+bool CTxMemPool::isSpent(const COutPoint& outpoint) const
 {
     LOCK(cs);
     return mapNextTx.count(outpoint);

src/txmempool.h

@@ -550,7 +550,7 @@ public:
     void _clear(); //lock free
     bool CompareDepthAndScore(const uint256& hasha, const uint256& hashb);
     void queryHashes(std::vector<uint256>& vtxid);
-    bool isSpent(const COutPoint& outpoint);
+    bool isSpent(const COutPoint& outpoint) const;
     unsigned int GetTransactionsUpdated() const;
     void AddTransactionsUpdated(unsigned int n);
     /**

test/functional/interface_rest.py

@@ -85,7 +85,7 @@ class RESTTest (BitcoinTestFramework):
         #######################################
         # GETUTXOS: query an unspent outpoint #
         #######################################
-        json_request = '/checkmempool/'+txid+'-'+str(n)
+        json_request = '/'+txid+'-'+str(n)
         json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
 
@@ -100,7 +100,7 @@ class RESTTest (BitcoinTestFramework):
         #################################################
         # GETUTXOS: now query an already spent outpoint #
         #################################################
-        json_request = '/checkmempool/'+vintx+'-0'
+        json_request = '/'+vintx+'-0'
         json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
 
@@ -117,7 +117,7 @@ class RESTTest (BitcoinTestFramework):
         ##################################################
         # GETUTXOS: now check both with the same request #
         ##################################################
-        json_request = '/checkmempool/'+txid+'-'+str(n)+'/'+vintx+'-0'
+        json_request = '/'+txid+'-'+str(n)+'/'+vintx+'-0'
         json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
         assert_equal(len(json_obj['utxos']), 1)
@@ -151,23 +151,48 @@ class RESTTest (BitcoinTestFramework):
         txid = self.nodes[0].sendtoaddress(self.nodes[1].getnewaddress(), 0.1)
         json_string = http_get_call(url.hostname, url.port, '/rest/tx/'+txid+self.FORMAT_SEPARATOR+"json")
         json_obj = json.loads(json_string)
-        vintx = json_obj['vin'][0]['txid'] # get the vin to later check for utxo (should be spent by then)
+        # get the spent output to later check for utxo (should be spent by then)
+        spent = '{}-{}'.format(json_obj['vin'][0]['txid'], json_obj['vin'][0]['vout'])
         # get n of 0.1 outpoint
         n = 0
         for vout in json_obj['vout']:
             if vout['value'] == 0.1:
                 n = vout['n']
+        spending = '{}-{}'.format(txid, n)
 
-        json_request = '/'+txid+'-'+str(n)
+        json_request = '/'+spending
         json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
-        assert_equal(len(json_obj['utxos']), 0) #there should be an outpoint because it has just added to the mempool
+        assert_equal(len(json_obj['utxos']), 0) #there should be no outpoint because it has just added to the mempool
 
-        json_request = '/checkmempool/'+txid+'-'+str(n)
+        json_request = '/checkmempool/'+spending
         json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
         json_obj = json.loads(json_string)
         assert_equal(len(json_obj['utxos']), 1) #there should be an outpoint because it has just added to the mempool
 
+        json_request = '/'+spent
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
+        json_obj = json.loads(json_string)
+        assert_equal(len(json_obj['utxos']), 1) #there should be an outpoint because its spending tx is not confirmed
+
+        json_request = '/checkmempool/'+spent
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
+        json_obj = json.loads(json_string)
+        assert_equal(len(json_obj['utxos']), 0) #there should be no outpoint because it has just spent (by mempool tx)
+
+        self.nodes[0].generate(1)
+        self.sync_all()
+
+        json_request = '/'+spending
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
+        json_obj = json.loads(json_string)
+        assert_equal(len(json_obj['utxos']), 1) #there should be an outpoint because it was mined
+
+        json_request = '/checkmempool/'+spending
+        json_string = http_get_call(url.hostname, url.port, '/rest/getutxos'+json_request+self.FORMAT_SEPARATOR+'json')
+        json_obj = json.loads(json_string)
+        assert_equal(len(json_obj['utxos']), 1) #there should be an outpoint because it was mined
+
         #do some invalid requests
         json_request = '{"checkmempool'
         response = http_post_call(url.hostname, url.port, '/rest/getutxos'+self.FORMAT_SEPARATOR+'json', json_request, True)