added scripts to generate parsable API docs (#187)

also included the generated docs
also changed Travis build script to not run the formatting check on OSX

.travis.yml

@@ -29,7 +29,7 @@ before_install:
 
 install: true
 script:
-  - if [[ "${TARGET}" == "osx" ]];     then ./reproducible_build.sh -t -o -c -f; fi
+  - if [[ "${TARGET}" == "osx" ]];     then ./reproducible_build.sh -t -o -c; fi
   - if [[ "${TARGET}" == "linux" ]];   then ./reproducible_build.sh -t -o -c -f; fi
   - if [[ "${TARGET}" == "windows" ]]; then ./packaging/build_windows.sh; fi
   - if [[ "${TARGET}" == "osx" ]];     then zip -j "lbrycrd-${TARGET}.zip" src/lbrycrdd src/lbrycrd-cli src/lbrycrd-tx; fi

CMakeLists.txt

@@ -1,5 +1,5 @@
 cmake_minimum_required(VERSION 3.7)
-project(lbrycrd_clion) # not expecting a full compile -- just something that allows clion syntax checking
+project(lbrycrd_clion) # Do not use for full compile. This is for CLion syntax checking only. 
 
 set (CMAKE_CXX_STANDARD 98) # 03 is not supported by cmake, but this will soon be 11 (after upstream bitcoin merge)
 # I thought that setting the standard would disable the clang-tidy c++11 tips; nope.
@@ -53,4 +53,4 @@ foreach(test ${tests})
     get_filename_component(filename ${test} NAME_WE)
     add_executable(${filename} ${test} ${sources})
     target_include_directories(${filename} PRIVATE src/test)
-endforeach(test)
+endforeach(test)

contrib/devtools/clang-format.py

@@ -12,21 +12,8 @@ import os
 import sys
 import subprocess
 
-tested_versions = ['3.6.0', '3.6.1', '3.6.2'] # A set of versions known to produce the same output
 accepted_file_extensions = ('.h', '.cpp') # Files to format
 
-def check_clang_format_version(clang_format_exe):
-    try:
-        output = subprocess.check_output([clang_format_exe, '-version'])
-        for ver in tested_versions:
-            if ver in output:
-                print "Detected clang-format version " + ver
-                return
-        raise RuntimeError("Untested version: " + output)
-    except Exception as e:
-        print 'Could not verify version of ' + clang_format_exe + '.'
-        raise e
-
 def check_command_line_args(argv):
     required_args = ['{clang-format-exe}', '{files}']
     example_args = ['clang-format-3.x', 'src/main.cpp', 'src/wallet/*']
@@ -46,7 +33,7 @@ def run_clang_format(clang_format_exe, files):
             for path, dirs, files in os.walk(target):
                 run_clang_format(clang_format_exe, (os.path.join(path, f) for f in files))
         elif target.endswith(accepted_file_extensions):
-            print "Format " + target
+            print "Formatting " + target
             subprocess.check_call([clang_format_exe, '-i', '-style=file', target], stdout=open(os.devnull, 'wb'), stderr=subprocess.STDOUT)
         else:
             print "Skip " + target
@@ -55,7 +42,6 @@ def main(argv):
     check_command_line_args(argv)
     clang_format_exe = argv[1]
     files = argv[2:]
-    check_clang_format_version(clang_format_exe)
     run_clang_format(clang_format_exe, files)
 
 if __name__ == "__main__":

contrib/devtools/generate_json_api_jrgen.py

@@ -0,0 +1,277 @@
+#!/usr/bin/env python3
+
+import re
+import subprocess as sp
+import sys
+import json
+import urllib.request as req
+import jsonschema
+
+
+re_full = re.compile(r'(?P<name>^.*?$)(?P<desc>.*?)(^Argument.*?$(?P<args>.*?))?(^Result[^\n,]*?:\s*$(?P<resl>.*?))?(^Exampl.*?$(?P<exmp>.*))?', re.DOTALL | re.MULTILINE)
+re_argline = re.compile(r'^("?)(?P<name>\w.*?)\1(\s*:.+?,?\s*)?\s+\((?P<type>.*?)\)\s*(?P<desc>.*?)\s*$', re.DOTALL)
+
+
+def get_obj_from_dirty_text(full_object: str):
+    lines = full_object.splitlines()
+    lefts = []
+    i = 0
+    while i < len(lines):
+        idx = lines[i].find('(')
+        left = lines[i][0:idx].strip() if idx >= 0 else lines[i]
+        left = left.rstrip('.')  # handling , ...
+        left = left.strip()
+        left = left.rstrip(',')
+        lefts.append(left)
+        while idx >= 0 and i < len(lines) - 1:
+            idx2 = len(re.match(r'^\s*', lines[i + 1]).group())
+            if idx2 > idx:
+                lines[i] += lines.pop(i + 1)[idx2 - 1:]
+            else:
+                break
+        i += 1
+
+    ret = None
+    try:
+        property_stack = []
+        object_stack = []
+        name_stack = []
+
+        last_name = None
+        for i in range(0, len(lines)):
+            left = lefts[i]
+            if not left:
+                continue
+            line = lines[i].strip()
+
+            arg_parsed = re_argline.fullmatch(line)
+            property_refined_type = 'object'
+            if arg_parsed is not None:
+                property_name, property_type, property_desc = arg_parsed.group('name', 'type', 'desc')
+                property_refined_type, property_required, property_child = get_type(property_type, None)
+
+                if property_refined_type is not 'array' and property_refined_type is not 'object':
+                    property_stack[-1][property_name] = {
+                        'type': property_refined_type,
+                        'description': property_desc
+                    }
+                else:
+                    last_name = property_name
+            elif len(left) > 1:
+                match = re.match(r'^(\[)?"(?P<name>\w.*?)"(\])?.*', left)
+                last_name = match.group('name')
+                if match.group(1) is not None and match.group(3) is not None:
+                    left = '['
+                    property_refined_type = 'string'
+                    if 'string' not in line:
+                        raise NotImplementedError('Not implemented: ' + line)
+
+            if left.endswith('['):
+                object_stack.append({'type': 'array', 'items': {'type': property_refined_type}})
+                property_stack.append({})
+                name_stack.append(last_name)
+            elif left.endswith('{'):
+                object_stack.append({'type': 'object'})
+                property_stack.append({})
+                name_stack.append(last_name)
+            elif (left.endswith(']') and '[' not in left) or (left.endswith('}') and '{' not in left):
+                obj = object_stack.pop()
+                prop = property_stack.pop()
+                name = name_stack.pop()
+                if len(prop) > 0:
+                    if 'items' in obj:
+                        obj['items']['properties'] = prop
+                    else:
+                        obj['properties'] = prop
+                if len(property_stack) > 0:
+                    if 'items' in object_stack[-1]:
+                        object_stack[-1]['items']['type'] = obj['type']
+                        if len(prop) > 0:
+                            object_stack[-1]['items']['properties'] = prop
+                    else:
+                        if name is None:
+                            raise RuntimeError('Not expected')
+                        property_stack[-1][name] = obj
+                else:
+                    ret = obj
+            if ret is not None:
+                if i + 1 < len(lines) - 1:
+                    print('Ignoring this data (below the parsed object): ' + "\n".join(lines[i+1:]), file=sys.stderr)
+                return ret
+    except Exception as e:
+        print('Exception: ' + str(e), file=sys.stderr)
+        print('Unable to cope with: ' + '\n'.join(lines), file=sys.stderr)
+    return None
+
+
+def get_type(arg_type: str, full_line: str):
+    if arg_type is None:
+        return 'string', True, None
+
+    required = 'required' in arg_type or 'optional' not in arg_type
+
+    arg_type = arg_type.lower()
+    if 'array' in arg_type:
+        return 'array', required, None
+    if 'numeric' in arg_type:
+        return 'number', required, None
+    if 'bool' in arg_type:
+        return 'boolean', required, None
+    if 'string' in arg_type:
+        return 'string', required, None
+    if 'object' in arg_type:
+        properties = get_obj_from_dirty_text(full_line) if full_line is not None else None
+        return 'object', required, properties
+
+    print('Unable to derive type from: ' + arg_type, file=sys.stderr)
+    return None, False, None
+
+
+def get_default(arg_refined_type: str, arg_type: str):
+    if 'default=' in arg_type:
+        if 'number' in arg_refined_type:
+            return int(re.match('.*default=([^,)]+)', arg_type).group(1))
+        if 'string' in arg_refined_type:
+            return re.match('.*default=([^,)]+)', arg_type).group(1)
+        if 'boolean' in arg_refined_type:
+            if 'default=true' in arg_type:
+                return True
+            if 'default=false' in arg_type:
+                return False
+            raise NotImplementedError('Not implemented: ' + arg_type)
+        if 'array' in arg_type:
+            raise NotImplementedError('Not implemented: ' + arg_type)
+    return None
+
+
+def parse_single_argument(line: str):
+    if line:
+        line = line.strip()
+    if not line or line.startswith('None'):
+        return None, None, False
+
+    arg_parsed = re_argline.fullmatch(line)
+    if arg_parsed is None:
+        if line.startswith('{') or line.startswith('['):
+            return get_obj_from_dirty_text(line), None, True
+        else:
+            print("Unparsable argument: " + line, file=sys.stderr)
+        descriptor = {
+            'type': 'array' if line.startswith('[') else 'object',
+            'description': line,
+        }
+        return descriptor, None, True
+
+    arg_name, arg_type, arg_desc = arg_parsed.group('name', 'type', 'desc')
+    if not arg_type:
+        raise NotImplementedError('Not implemented: ' + arg_type)
+    arg_refined_type, arg_required, arg_properties = get_type(arg_type, arg_desc)
+
+    if arg_properties is not None:
+        return arg_properties, arg_name, arg_required
+
+    arg_refined_default = get_default(arg_refined_type, arg_type)
+    arg_desc = re.sub('\s+', ' ', arg_desc.strip()) \
+        if arg_desc and arg_refined_type is not 'object' and arg_refined_type is not 'array' \
+        else arg_desc.strip() if arg_desc else ''
+
+    descriptor = {
+        'type': arg_refined_type,
+        'description': arg_desc,
+    }
+    if arg_refined_default is not None:
+        descriptor['default'] = arg_refined_default
+    return descriptor, arg_name, arg_required
+
+
+def parse_params(args: str):
+    arguments = {}
+    requireds = []
+    if args:
+        for line in re.split('\s*\d+\.\s+', args, re.DOTALL):
+            descriptor, name, required = parse_single_argument(line)
+            if descriptor is None:
+                continue
+            if required:
+                requireds.append(name)
+            arguments[name] = descriptor
+    return arguments, requireds
+
+
+def get_api(section_name: str, command: str, command_help: str):
+
+    parsed = re_full.fullmatch(command_help)
+    if parsed is None:
+        raise RuntimeError('Unable to resolve help format for ' + command)
+
+    name, desc, args, resl, exmp = parsed.group('name', 'desc', 'args', 'resl', 'exmp')
+
+    properties, required = parse_params(args)
+    result_descriptor, result_name, result_required = parse_single_argument(resl)
+
+    desc = re.sub('\s+', ' ', desc.strip()) if desc else name
+    example_array = exmp.splitlines() if exmp else []
+
+    ret = {
+        'summary': desc,
+        'description': example_array,
+        'tags': [section_name],
+        'params': {
+            'type': 'object',
+            'properties': properties,
+            'required': required
+        },
+    }
+    if result_descriptor is not None:
+        ret['result'] = result_descriptor
+    return ret
+
+
+def write_api():
+    if len(sys.argv) < 2:
+        print("Missing required argument: <path to CLI tool>", file=sys.stderr)
+        sys.exit(1)
+    cli_tool = sys.argv[1]
+    result = sp.run([cli_tool, "help"], stdout=sp.PIPE, universal_newlines=True)
+    commands = result.stdout
+    sections = re.split('^==\s*(.*?)\s*==$', commands, flags=re.MULTILINE)
+    methods = {}
+    for section in sections:
+        if not section:
+            continue
+        lines = section.splitlines()
+        if len(lines) == 1:
+            section_name = lines[0]
+            continue
+        for command in sorted(lines[1:]):
+            if not command:
+                continue
+            command = command.split(' ')[0]
+            result = sp.run([cli_tool, "help", command], stdout=sp.PIPE, universal_newlines=True)
+            methods[command] = get_api(section_name, command, result.stdout)
+
+    version = sp.run([cli_tool, "--version"], stdout=sp.PIPE, universal_newlines=True)
+    wrapper = {
+        '$schema': 'https://rawgit.com/mzernetsch/jrgen/master/jrgen-spec.schema.json',
+        'jrgen': '1.1',
+        'jsonrpc': '1.0',  # see https://github.com/bitcoin/bitcoin/pull/12435
+        'info': {
+            'title': 'lbrycrd RPC API',
+            'version': version.stdout.strip(),
+            'description': []
+        },
+        'definitions': {},  # for items used in $ref further down
+        'methods': methods,
+    }
+
+    schema = req.urlopen(wrapper['$schema']).read().decode('utf-8')
+    try:
+        jsonschema.validate(wrapper, schema)
+    except Exception as e:
+        print('From schema validation: ' + str(e), file=sys.stderr)
+
+    print(json.dumps(wrapper, indent=4))
+
+
+if __name__ == '__main__':
+    write_api()

contrib/devtools/generate_json_api_v1.py

@@ -0,0 +1,106 @@
+#!/usr/bin/env python3
+
+import re
+import subprocess as sp
+import sys
+import json
+
+re_full = re.compile(r'(?P<name>^.*?$)(?P<desc>.*?)(^Argument.*?$(?P<args>.*?))?(^Result[^\n]*?:\s*$(?P<resl>.*?))?(^Exampl.*?$(?P<exmp>.*))?', re.DOTALL | re.MULTILINE)
+re_argline = re.compile(r'^("?)(?P<name>.*?)\1\s+\((?P<type>.*?)\)\s*(?P<desc>.*)$', re.DOTALL)
+
+
+def get_type(arg_type, full_line):
+    if arg_type is None:
+        return 'string'
+
+    arg_type = arg_type.lower()
+    if 'numeric' in arg_type:
+        return 'number'
+    if 'bool' in arg_type:
+        return 'boolean'
+    if 'string' in arg_type:
+        return 'string'
+    if 'object' in arg_type:
+        return 'object'
+
+    raise Exception('Not implemented: ' + arg_type)
+
+
+def parse_params(args):
+    arguments = []
+    if args:
+        for line in re.split('\s*\d+\.\s+', args, re.DOTALL):
+            if not line or not line.strip() or line.strip().startswith('None'):
+                continue
+            arg_parsed = re_argline.fullmatch(line)
+            if arg_parsed is None:
+                raise Exception("Unparsable argument: " + line)
+            arg_name, arg_type, arg_desc = arg_parsed.group('name', 'type', 'desc')
+            if not arg_type:
+                raise Exception('Not implemented: ' + arg_type)
+            arg_required = 'required' in arg_type or 'optional' not in arg_type
+            arg_refined_type = get_type(arg_type, line)
+            arg_desc = re.sub('\s+', ' ', arg_desc.strip()) if arg_desc else []
+            arguments.append({
+                'name': arg_name,
+                'type': arg_refined_type,
+                'description': arg_desc,
+                'is_required': arg_required
+            })
+    return arguments
+
+
+def get_api(section_name, command, command_help):
+
+    parsed = re_full.fullmatch(command_help)
+    if parsed is None:
+        raise Exception('Unable to resolve help format for ' + command)
+
+    name, desc, args, resl, exmp = parsed.group('name', 'desc', 'args', 'resl', 'exmp')
+
+    arguments = parse_params(args)
+
+    cmd_desc = re.sub('\s+', ' ', desc.strip()) if desc else ''
+    if exmp and '--skip_examples' not in sys.argv:
+        cmd_desc += '\nExamples:\n' + exmp.strip()
+    cmd_resl = resl.strip() if resl else None
+
+    ret = {
+        'name': command,
+        'namespace': section_name,
+        'description': cmd_desc,
+        'arguments': arguments,
+    }
+    if cmd_resl is not None:
+        ret['returns'] = cmd_resl
+    return ret
+
+
+def write_api():
+    if len(sys.argv) < 2:
+        print("Missing required argument: <path to CLI tool>", file=sys.stderr)
+        sys.exit(1)
+    cli_tool = sys.argv[1]
+    result = sp.run([cli_tool, "help"], stdout=sp.PIPE, universal_newlines=True)
+    commands = result.stdout
+    sections = re.split('^==\s*(.*?)\s*==$', commands, flags=re.MULTILINE)
+    apis = []
+    for section in sections:
+        if not section:
+            continue
+        lines = section.splitlines()
+        if len(lines) == 1:
+            section_name = lines[0]
+            continue
+        for command in sorted(lines[1:]):
+            if not command:
+                continue
+            command = command.split(' ')[0]
+            result = sp.run([cli_tool, "help", command], stdout=sp.PIPE, universal_newlines=True)
+            apis.append(get_api(section_name, command, result.stdout))
+
+    print(json.dumps(apis, indent=4))
+
+
+if __name__ == '__main__':
+    write_api()

src/rpc/claimtrie.cpp

@@ -369,31 +369,30 @@ UniValue getclaimbyid(const UniValue& params, bool fHelp)
 {
     if (fHelp || params.size() != 1)
         throw std::runtime_error(
-                "getclaimbyid\n"
-                        "Get a claim by claim id\n"
-                        "Arguments: \n"
-                        "1.  \"claimId\"           (string) the claimId of this claim\n"
-                        "Result:\n"
-                        "{\n"
-                        "  \"name\"                (string) the name of the claim\n"
-                        "  \"value\"               (string) claim metadata\n"
-                        "  \"claimId\"             (string) the claimId of this claim\n"
-                        "  \"txid\"                (string) the hash of the transaction which has successfully claimed this name\n"
-                        "  \"n\"                   (numeric) vout value\n"
-                        "  \"amount\"              (numeric) txout value\n"
-                        "  \"effective amount\"    (numeric) txout amount plus amount from all supports associated with the claim\n"
-                        "  \"height\"              (numeric) the height of the block in which this claim transaction is located\n"
-                        "  \"supports\"            (array of object) supports for this claim\n"
-                        "  \"valid at height\"     (numeric) the height at which the claim is valid\n"
-                        "  [\n"
-                        "    \"txid\"              (string) the txid of the support\n"
-                        "    \"n\"                 (numeric) the index of the support in the transaction's list of outputs\n"
-                        "    \"height\"            (numeric) the height at which the support was included in the blockchain\n"
-                        "    \"valid at height\"   (numeric) the height at which the support is valid\n"
-                        "    \"amount\"            (numeric) the amount of the support\n"
-                        "  ]\n"
-                        "}\n"
-        );
+            "getclaimbyid\n"
+            "Get a claim by claim id\n"
+            "Arguments: \n"
+            "1.  \"claimId\"           (string) the claimId of this claim\n"
+            "Result:\n"
+            "{\n"
+            "  \"name\"                (string) the name of the claim\n"
+            "  \"value\"               (string) claim metadata\n"
+            "  \"claimId\"             (string) the claimId of this claim\n"
+            "  \"txid\"                (string) the hash of the transaction which has successfully claimed this name\n"
+            "  \"n\"                   (numeric) vout value\n"
+            "  \"amount\"              (numeric) txout value\n"
+            "  \"effective amount\"    (numeric) txout amount plus amount from all supports associated with the claim\n"
+            "  \"supports\"            (array of object) supports for this claim\n"
+            "  [\n"
+            "    \"txid\"              (string) the txid of the support\n"
+            "    \"n\"                 (numeric) the index of the support in the transaction's list of outputs\n"
+            "    \"height\"            (numeric) the height at which the support was included in the blockchain\n"
+            "    \"valid at height\"   (numeric) the height at which the support is valid\n"
+            "    \"amount\"            (numeric) the amount of the support\n"
+            "  ]\n"
+            "  \"height\"              (numeric) the height of the block in which this claim transaction is located\n"
+            "  \"valid at height\"     (numeric) the height at which the claim is valid\n"
+            "}\n");
 
     LOCK(cs_main);
     uint160 claimId = ParseClaimtrieId(params[0], "Claim-id (parameter 1)");