Merge pull request #3695

b5ad5e7 Add Python test for -rpcbind and -rpcallowip (Wladimir J. van der Laan)
f923c07 Support IPv6 lookup in bitcoin-cli even when IPv6 only bound on localhost (Wladimir J. van der Laan)
deb3572 Add -rpcbind option to allow binding RPC port on a specific interface (Wladimir J. van der Laan)

qa/rpc-tests/netutil.py

@@ -0,0 +1,134 @@
+# Linux network utilities
+import sys
+import socket
+import fcntl
+import struct
+import array
+import os
+import binascii
+
+# Roughly based on http://voorloopnul.com/blog/a-python-netstat-in-less-than-100-lines-of-code/ by Ricardo Pascal
+STATE_ESTABLISHED = '01'
+STATE_SYN_SENT  = '02'
+STATE_SYN_RECV = '03'
+STATE_FIN_WAIT1 = '04'
+STATE_FIN_WAIT2 = '05'
+STATE_TIME_WAIT = '06'
+STATE_CLOSE = '07'
+STATE_CLOSE_WAIT = '08'
+STATE_LAST_ACK = '09'
+STATE_LISTEN = '0A'
+STATE_CLOSING = '0B'
+
+def get_socket_inodes(pid):
+    '''
+    Get list of socket inodes for process pid.
+    '''
+    base = '/proc/%i/fd' % pid
+    inodes = []
+    for item in os.listdir(base):
+        target = os.readlink(os.path.join(base, item))
+        if target.startswith('socket:'):
+            inodes.append(int(target[8:-1]))
+    return inodes
+
+def _remove_empty(array):
+    return [x for x in array if x !='']
+
+def _convert_ip_port(array):
+    host,port = array.split(':')
+    # convert host from mangled-per-four-bytes form as used by kernel
+    host = binascii.unhexlify(host)
+    host_out = ''
+    for x in range(0, len(host)/4):
+        (val,) = struct.unpack('=I', host[x*4:(x+1)*4])
+        host_out += '%08x' % val
+
+    return host_out,int(port,16)
+
+def netstat(typ='tcp'):
+    '''
+    Function to return a list with status of tcp connections at linux systems
+    To get pid of all network process running on system, you must run this script
+    as superuser
+    '''
+    with open('/proc/net/'+typ,'r') as f:
+        content = f.readlines()
+        content.pop(0)
+    result = []
+    for line in content:
+        line_array = _remove_empty(line.split(' '))     # Split lines and remove empty spaces.
+        tcp_id = line_array[0]
+        l_addr = _convert_ip_port(line_array[1])
+        r_addr = _convert_ip_port(line_array[2])
+        state = line_array[3]
+        inode = int(line_array[9])                      # Need the inode to match with process pid.
+        nline = [tcp_id, l_addr, r_addr, state, inode]
+        result.append(nline)
+    return result
+
+def get_bind_addrs(pid):
+    '''
+    Get bind addresses as (host,port) tuples for process pid.
+    '''
+    inodes = get_socket_inodes(pid)
+    bind_addrs = []
+    for conn in netstat('tcp') + netstat('tcp6'):
+        if conn[3] == STATE_LISTEN and conn[4] in inodes:
+            bind_addrs.append(conn[1])
+    return bind_addrs
+
+# from: http://code.activestate.com/recipes/439093/
+def all_interfaces():
+    '''
+    Return all interfaces that are up
+    '''
+    is_64bits = sys.maxsize > 2**32
+    struct_size = 40 if is_64bits else 32
+    s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+    max_possible = 8 # initial value
+    while True:
+        bytes = max_possible * struct_size
+        names = array.array('B', '\0' * bytes)
+        outbytes = struct.unpack('iL', fcntl.ioctl(
+            s.fileno(),
+            0x8912,  # SIOCGIFCONF
+            struct.pack('iL', bytes, names.buffer_info()[0])
+        ))[0]
+        if outbytes == bytes:
+            max_possible *= 2
+        else:
+            break
+    namestr = names.tostring()
+    return [(namestr[i:i+16].split('\0', 1)[0],
+             socket.inet_ntoa(namestr[i+20:i+24]))
+            for i in range(0, outbytes, struct_size)]
+
+def addr_to_hex(addr):
+    '''
+    Convert string IPv4 or IPv6 address to binary address as returned by
+    get_bind_addrs.
+    Very naive implementation that certainly doesn't work for all IPv6 variants.
+    '''
+    if '.' in addr: # IPv4
+        addr = [int(x) for x in addr.split('.')]
+    elif ':' in addr: # IPv6
+        sub = [[], []] # prefix, suffix
+        x = 0
+        addr = addr.split(':')
+        for i,comp in enumerate(addr):
+            if comp == '':
+                if i == 0 or i == (len(addr)-1): # skip empty component at beginning or end
+                    continue
+                x += 1 # :: skips to suffix
+                assert(x < 2)
+            else: # two bytes per component
+                val = int(comp, 16)
+                sub[x].append(val >> 8)
+                sub[x].append(val & 0xff)
+        nullbytes = 16 - len(sub[0]) - len(sub[1])
+        assert((x == 0 and nullbytes == 0) or (x == 1 and nullbytes > 0))
+        addr = sub[0] + ([0] * nullbytes) + sub[1]
+    else:
+        raise ValueError('Could not parse address %s' % addr)
+    return binascii.hexlify(bytearray(addr))

qa/rpc-tests/rpcbind_test.py

@@ -0,0 +1,152 @@
+#!/usr/bin/env python
+# Copyright (c) 2014 The Bitcoin Core developers
+# Distributed under the MIT/X11 software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+# Test for -rpcbind, as well as -rpcallowip and -rpcconnect
+
+# Add python-bitcoinrpc to module search path:
+import os
+import sys
+sys.path.append(os.path.join(os.path.dirname(os.path.abspath(__file__)), "python-bitcoinrpc"))
+
+import json
+import shutil
+import subprocess
+import tempfile
+import traceback
+
+from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
+from util import *
+from netutil import *
+
+def run_bind_test(tmpdir, allow_ips, connect_to, addresses, expected):
+    '''
+    Start a node with requested rpcallowip and rpcbind parameters,
+    then try to connect, and check if the set of bound addresses
+    matches the expected set.
+    '''
+    expected = [(addr_to_hex(addr), port) for (addr, port) in expected]
+    base_args = ['-disablewallet', '-nolisten']
+    if allow_ips:
+        base_args += ['-rpcallowip=' + x for x in allow_ips]
+    binds = ['-rpcbind='+addr for addr in addresses]
+    nodes = start_nodes(1, tmpdir, [base_args + binds], connect_to)
+    try:
+        pid = bitcoind_processes[0].pid
+        assert_equal(set(get_bind_addrs(pid)), set(expected))
+    finally:
+        stop_nodes(nodes)
+        wait_bitcoinds()
+
+def run_allowip_test(tmpdir, allow_ips, rpchost):
+    '''
+    Start a node with rpcwallow IP, and request getinfo
+    at a non-localhost IP.
+    '''
+    base_args = ['-disablewallet', '-nolisten'] + ['-rpcallowip='+x for x in allow_ips]
+    nodes = start_nodes(1, tmpdir, [base_args])
+    try:
+        # connect to node through non-loopback interface
+        url = "http://rt:rt@%s:%d" % (rpchost, START_RPC_PORT,)
+        node = AuthServiceProxy(url)
+        node.getinfo()
+    finally:
+        node = None # make sure connection will be garbage collected and closed
+        stop_nodes(nodes)
+        wait_bitcoinds()
+
+
+def run_test(tmpdir):
+    assert(sys.platform == 'linux2') # due to OS-specific network stats queries, this test works only on Linux
+    # find the first non-loopback interface for testing
+    non_loopback_ip = None
+    for name,ip in all_interfaces():
+        if ip != '127.0.0.1':
+            non_loopback_ip = ip
+            break
+    if non_loopback_ip is None:
+        assert(not 'This test requires at least one non-loopback IPv4 interface')
+    print("Using interface %s for testing" % non_loopback_ip)
+
+    # check default without rpcallowip (IPv4 and IPv6 localhost)
+    run_bind_test(tmpdir, None, '127.0.0.1', [],
+        [('127.0.0.1', 11100), ('::1', 11100)])
+    # check default with rpcallowip (IPv6 any)
+    run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1', [],
+        [('::0', 11100)])
+    # check only IPv4 localhost (explicit)
+    run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1', ['127.0.0.1'],
+        [('127.0.0.1', START_RPC_PORT)])
+    # check only IPv4 localhost (explicit) with alternative port
+    run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171'],
+        [('127.0.0.1', 32171)])
+    # check only IPv4 localhost (explicit) with multiple alternative ports on same host
+    run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1:32171', ['127.0.0.1:32171', '127.0.0.1:32172'],
+        [('127.0.0.1', 32171), ('127.0.0.1', 32172)])
+    # check only IPv6 localhost (explicit)
+    run_bind_test(tmpdir, ['[::1]'], '[::1]', ['[::1]'],
+        [('::1', 11100)])
+    # check both IPv4 and IPv6 localhost (explicit)
+    run_bind_test(tmpdir, ['127.0.0.1'], '127.0.0.1', ['127.0.0.1', '[::1]'],
+        [('127.0.0.1', START_RPC_PORT), ('::1', START_RPC_PORT)])
+    # check only non-loopback interface
+    run_bind_test(tmpdir, [non_loopback_ip], non_loopback_ip, [non_loopback_ip],
+        [(non_loopback_ip, START_RPC_PORT)])
+
+    # Check that with invalid rpcallowip, we are denied
+    run_allowip_test(tmpdir, [non_loopback_ip], non_loopback_ip)
+    try:
+        run_allowip_test(tmpdir, ['1.1.1.1'], non_loopback_ip)
+        assert(not 'Connection not denied by rpcallowip as expected')
+    except ValueError:
+        pass
+
+def main():
+    import optparse
+
+    parser = optparse.OptionParser(usage="%prog [options]")
+    parser.add_option("--nocleanup", dest="nocleanup", default=False, action="store_true",
+                      help="Leave bitcoinds and test.* datadir on exit or error")
+    parser.add_option("--srcdir", dest="srcdir", default="../../src",
+                      help="Source directory containing bitcoind/bitcoin-cli (default: %default%)")
+    parser.add_option("--tmpdir", dest="tmpdir", default=tempfile.mkdtemp(prefix="test"),
+                      help="Root directory for datadirs")
+    (options, args) = parser.parse_args()
+
+    os.environ['PATH'] = options.srcdir+":"+os.environ['PATH']
+
+    check_json_precision()
+
+    success = False
+    nodes = []
+    try:
+        print("Initializing test directory "+options.tmpdir)
+        if not os.path.isdir(options.tmpdir):
+            os.makedirs(options.tmpdir)
+        initialize_chain(options.tmpdir)
+
+        run_test(options.tmpdir)
+
+        success = True
+
+    except AssertionError as e:
+        print("Assertion failed: "+e.message)
+    except Exception as e:
+        print("Unexpected exception caught during testing: "+str(e))
+        traceback.print_tb(sys.exc_info()[2])
+
+    if not options.nocleanup:
+        print("Cleaning up")
+        wait_bitcoinds()
+        shutil.rmtree(options.tmpdir)
+
+    if success:
+        print("Tests successful")
+        sys.exit(0)
+    else:
+        print("Failed")
+        sys.exit(1)
+
+if __name__ == '__main__':
+    main()

qa/rpc-tests/util.py

@@ -15,6 +15,7 @@ import json
 import shutil
 import subprocess
 import time
+import re
 
 from bitcoinrpc.authproxy import AuthServiceProxy, JSONRPCException
 from util import *
@@ -112,20 +113,43 @@ def initialize_chain(test_dir):
         to_dir = os.path.join(test_dir,  "node"+str(i))
         shutil.copytree(from_dir, to_dir)
 
-def start_nodes(num_nodes, dir):
+def _rpchost_to_args(rpchost):
+    '''Convert optional IP:port spec to rpcconnect/rpcport args'''
+    if rpchost is None:
+        return []
+
+    match = re.match('(\[[0-9a-fA-f:]+\]|[^:]+)(?::([0-9]+))?$', rpchost)
+    if not match:
+        raise ValueError('Invalid RPC host spec ' + rpchost)
+
+    rpcconnect = match.group(1)
+    rpcport = match.group(2)
+
+    if rpcconnect.startswith('['): # remove IPv6 [...] wrapping
+        rpcconnect = rpcconnect[1:-1]
+
+    rv = ['-rpcconnect=' + rpcconnect]
+    if rpcport:
+        rv += ['-rpcport=' + rpcport]
+    return rv
+
+def start_nodes(num_nodes, dir, extra_args=None, rpchost=None):
     # Start bitcoinds, and wait for RPC interface to be up and running:
     devnull = open("/dev/null", "w+")
     for i in range(num_nodes):
         datadir = os.path.join(dir, "node"+str(i))
         args = [ "bitcoind", "-datadir="+datadir ]
+        if extra_args is not None:
+            args += extra_args[i]
         bitcoind_processes.append(subprocess.Popen(args))
-        subprocess.check_call([ "bitcoin-cli", "-datadir="+datadir,
+        subprocess.check_call([ "bitcoin-cli", "-datadir="+datadir] +
-                                  "-rpcwait", "getblockcount"], stdout=devnull)
+                                  _rpchost_to_args(rpchost)  +
+                                  ["-rpcwait", "getblockcount"], stdout=devnull)
     devnull.close()
     # Create&return JSON-RPC connections
     rpc_connections = []
     for i in range(num_nodes):
-        url = "http://rt:rt@127.0.0.1:%d"%(START_RPC_PORT+i,)
+        url = "http://rt:rt@%s:%d" % (rpchost or '127.0.0.1', START_RPC_PORT+i,)
         rpc_connections.append(AuthServiceProxy(url))
     return rpc_connections
 

src/init.cpp

@@ -304,10 +304,11 @@ std::string HelpMessage(HelpMessageMode hmm)
 
     strUsage += "\n" + _("RPC server options:") + "\n";
     strUsage += "  -server                " + _("Accept command line and JSON-RPC commands") + "\n";
+    strUsage += "  -rpcbind=<addr>        " + _("Bind to given address to listen for JSON-RPC connections. Use [host]:port notation for IPv6. This option can be specified multiple times (default: bind to all interfaces)") + "\n";
     strUsage += "  -rpcuser=<user>        " + _("Username for JSON-RPC connections") + "\n";
     strUsage += "  -rpcpassword=<pw>      " + _("Password for JSON-RPC connections") + "\n";
     strUsage += "  -rpcport=<port>        " + _("Listen for JSON-RPC connections on <port> (default: 8332 or testnet: 18332)") + "\n";
-    strUsage += "  -rpcallowip=<ip>       " + _("Allow JSON-RPC connections from specified IP address") + "\n";
+    strUsage += "  -rpcallowip=<ip>       " + _("Allow JSON-RPC connections from specified IP address. This option can be specified multiple times") + "\n";
     strUsage += "  -rpcthreads=<n>        " + _("Set the number of threads to service RPC calls (default: 4)") + "\n";
 
     strUsage += "\n" + _("RPC SSL options: (see the Bitcoin Wiki for SSL setup instructions)") + "\n";

src/rpcprotocol.h

@@ -103,11 +103,27 @@ public:
     }
     bool connect(const std::string& server, const std::string& port)
     {
-        boost::asio::ip::tcp::resolver resolver(stream.get_io_service());
+        using namespace boost::asio::ip;
-        boost::asio::ip::tcp::resolver::query query(server.c_str(), port.c_str());
+        tcp::resolver resolver(stream.get_io_service());
-        boost::asio::ip::tcp::resolver::iterator endpoint_iterator = resolver.resolve(query);
+        tcp::resolver::iterator endpoint_iterator;
-        boost::asio::ip::tcp::resolver::iterator end;
+#if BOOST_VERSION >= 104300
+        try {
+#endif
+            // The default query (flags address_configured) tries IPv6 if
+            // non-localhost IPv6 configured, and IPv4 if non-localhost IPv4
+            // configured.
+            tcp::resolver::query query(server.c_str(), port.c_str());
+            endpoint_iterator = resolver.resolve(query);
+#if BOOST_VERSION >= 104300
+        } catch(boost::system::system_error &e)
+        {
+            // If we at first don't succeed, try blanket lookup (IPv4+IPv6 independent of configured interfaces)
+            tcp::resolver::query query(server.c_str(), port.c_str(), resolver_query_base::flags());
+            endpoint_iterator = resolver.resolve(query);
+        }
+#endif
         boost::system::error_code error = boost::asio::error::host_not_found;
+        tcp::resolver::iterator end;
         while (error && endpoint_iterator != end)
         {
             stream.lowest_layer().close();

src/rpcserver.cpp

@@ -508,6 +508,14 @@ static void RPCAcceptHandler(boost::shared_ptr< basic_socket_acceptor<Protocol,
     }
 }
 
+static ip::tcp::endpoint ParseEndpoint(const std::string &strEndpoint, int defaultPort)
+{
+    std::string addr;
+    int port = defaultPort;
+    SplitHostPort(strEndpoint, port, addr);
+    return ip::tcp::endpoint(asio::ip::address::from_string(addr), port);
+}
+
 void StartRPCThreads()
 {
     rpc_allow_subnets.clear();
@@ -589,57 +597,74 @@ void StartRPCThreads()
         SSL_CTX_set_cipher_list(rpc_ssl_context->impl(), strCiphers.c_str());
     }
 
-    // Try a dual IPv6/IPv4 socket, falling back to separate IPv4 and IPv6 sockets
+    std::vector<ip::tcp::endpoint> vEndpoints;
-    const bool loopback = !mapArgs.count("-rpcallowip");
+    bool bBindAny = false;
-    asio::ip::address bindAddress = loopback ? asio::ip::address_v6::loopback() : asio::ip::address_v6::any();
+    int defaultPort = GetArg("-rpcport", Params().RPCPort());
-    ip::tcp::endpoint endpoint(bindAddress, GetArg("-rpcport", Params().RPCPort()));
+    if (!mapArgs.count("-rpcallowip")) // Default to loopback if not allowing external IPs
-    boost::system::error_code v6_only_error;
+    {
+        vEndpoints.push_back(ip::tcp::endpoint(asio::ip::address_v6::loopback(), defaultPort));
+        vEndpoints.push_back(ip::tcp::endpoint(asio::ip::address_v4::loopback(), defaultPort));
+        if (mapArgs.count("-rpcbind"))
+        {
+            LogPrintf("WARNING: option -rpcbind was ignored because -rpcallowip was not specified, refusing to allow everyone to connect\n");
+        }
+    } else if (mapArgs.count("-rpcbind")) // Specific bind address
+    {
+        BOOST_FOREACH(const std::string &addr, mapMultiArgs["-rpcbind"])
+        {
+            try {
+                vEndpoints.push_back(ParseEndpoint(addr, defaultPort));
+            }
+            catch(boost::system::system_error &e)
+            {
+                uiInterface.ThreadSafeMessageBox(
+                    strprintf(_("Could not parse -rpcbind value %s as network address"), addr),
+                    "", CClientUIInterface::MSG_ERROR);
+                StartShutdown();
+                return;
+            }
+        }
+    } else { // No specific bind address specified, bind to any
+        vEndpoints.push_back(ip::tcp::endpoint(asio::ip::address_v6::any(), defaultPort));
+        vEndpoints.push_back(ip::tcp::endpoint(asio::ip::address_v4::any(), defaultPort));
+        // Prefer making the socket dual IPv6/IPv4 instead of binding
+        // to both addresses seperately.
+        bBindAny = true;
+    }
 
     bool fListening = false;
     std::string strerr;
-    try
+    BOOST_FOREACH(const ip::tcp::endpoint &endpoint, vEndpoints)
     {
+        asio::ip::address bindAddress = endpoint.address();
+        LogPrintf("Binding RPC on address %s port %i (IPv4+IPv6 bind any: %i)\n", bindAddress.to_string(), endpoint.port(), bBindAny);
+        boost::system::error_code v6_only_error;
         boost::shared_ptr<ip::tcp::acceptor> acceptor(new ip::tcp::acceptor(*rpc_io_service));
         rpc_acceptors.push_back(acceptor);
-        acceptor->open(endpoint.protocol());
-        acceptor->set_option(boost::asio::ip::tcp::acceptor::reuse_address(true));
 
-        // Try making the socket dual IPv6/IPv4 (if listening on the "any" address)
+        try {
-        acceptor->set_option(boost::asio::ip::v6_only(loopback), v6_only_error);
-
-        acceptor->bind(endpoint);
-        acceptor->listen(socket_base::max_connections);
-
-        RPCListen(acceptor, *rpc_ssl_context, fUseSSL);
-
-        fListening = true;
-    }
-    catch(boost::system::system_error &e)
-    {
-        strerr = strprintf(_("An error occurred while setting up the RPC port %u for listening on IPv6, falling back to IPv4: %s"), endpoint.port(), e.what());
-    }
-    try {
-        // If dual IPv6/IPv4 failed (or we're opening loopback interfaces only), open IPv4 separately
-        if (!fListening || loopback || v6_only_error)
-        {
-            bindAddress = loopback ? asio::ip::address_v4::loopback() : asio::ip::address_v4::any();
-            endpoint.address(bindAddress);
-
-            boost::shared_ptr<ip::tcp::acceptor> acceptor(new ip::tcp::acceptor(*rpc_io_service));
-            rpc_acceptors.push_back(acceptor);
             acceptor->open(endpoint.protocol());
             acceptor->set_option(boost::asio::ip::tcp::acceptor::reuse_address(true));
+
+            // Try making the socket dual IPv6/IPv4 when listening on the IPv6 "any" address
+            acceptor->set_option(boost::asio::ip::v6_only(
+                !bBindAny || bindAddress != asio::ip::address_v6::any()), v6_only_error);
+
             acceptor->bind(endpoint);
             acceptor->listen(socket_base::max_connections);
 
             RPCListen(acceptor, *rpc_ssl_context, fUseSSL);
 
             fListening = true;
+            // If dual IPv6/IPv4 bind succesful, skip binding to IPv4 separately
+            if(bBindAny && bindAddress == asio::ip::address_v6::any() && !v6_only_error)
+                break;
+        }
+        catch(boost::system::system_error &e)
+        {
+            LogPrintf("ERROR: Binding RPC on address %s port %i failed: %s\n", bindAddress.to_string(), endpoint.port(), e.what());
+            strerr = strprintf(_("An error occurred while setting up the RPC address %s port %u for listening: %s"), bindAddress.to_string(), endpoint.port(), e.what());
         }
-    }
-    catch(boost::system::system_error &e)
-    {
-        strerr = strprintf(_("An error occurred while setting up the RPC port %u for listening on IPv4: %s"), endpoint.port(), e.what());
     }
 
     if (!fListening) {