Merge branch 'docker'

* docker:
  fix gitlab ci
  add min_version and method name to metrics
  include docker tag in build info
  build_type.py -> build_info.py
  add docker build info to wallet server image

.gitlab-ci.yml

@@ -72,8 +72,8 @@ test:json-api:
   script:
     - pip install --upgrade 'setuptools<45.0.0'
     - pip install pyinstaller
-    - python3.7 docker/set_build.py
     - pip install -e .
+    - python3.7 docker/set_build.py # must come after lbry is installed because it imports lbry
     - pyinstaller --onefile --name lbrynet lbry/extras/cli.py
     - chmod +x dist/lbrynet
     - zip --junk-paths ${CI_PROJECT_DIR}/lbrynet-${OS}.zip dist/lbrynet  # gitlab expects artifacts to be in $CI_PROJECT_DIR
@@ -129,8 +129,8 @@ build:windows:
   script:
     - pip install --upgrade 'setuptools<45.0.0'
     - pip install pyinstaller==3.5
-    - python docker/set_build.py
     - pip install -e .
+    - python docker/set_build.py # must come after lbry is installed because it imports lbry
     - pyinstaller --additional-hooks-dir=scripts/. --icon=icons/lbry256.ico -F -n lbrynet lbry/extras/cli.py
     - 7z a -tzip $env:CI_PROJECT_DIR/lbrynet-${OS}.zip ./dist/lbrynet.exe
     - checksum --type=sha256 --file=$env:CI_PROJECT_DIR/lbrynet-${OS}.zip

docker/Dockerfile.wallet_server

@@ -4,6 +4,10 @@ ARG user=lbry
 ARG db_dir=/database
 ARG projects_dir=/home/$user
 
+ARG DOCKER_TAG
+ARG DOCKER_COMMIT=docker
+ENV DOCKER_TAG=$DOCKER_TAG DOCKER_COMMIT=$DOCKER_COMMIT
+
 RUN apt-get update && \
     apt-get -y --no-install-recommends install \
       wget \
@@ -29,6 +33,7 @@ WORKDIR $projects_dir
 
 RUN pip install uvloop
 RUN make install
+RUN python3 docker/set_build.py
 RUN rm ~/.cache -rf
 
 # entry point

docker/hooks/build

@@ -0,0 +1,7 @@
+#!/bin/bash
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd "$DIR/../.." ## make sure we're in the right place. Docker Hub screws this up sometimes
+echo "docker build dir: $(pwd)"
+
+docker build --build-arg DOCKER_TAG=$DOCKER_TAG --build-arg DOCKER_COMMIT=$SOURCE_COMMIT -f $DOCKERFILE_PATH -t $IMAGE_NAME .

docker/set_build.py

@@ -1,38 +1,41 @@
-"""Set the build version to be 'qa', 'rc', 'release'"""
-
 import sys
 import os
 import re
 import logging
+import lbry.build_info as build_info_mod
 
 log = logging.getLogger()
 log.addHandler(logging.StreamHandler())
 log.setLevel(logging.DEBUG)
 
 
-def get_build_type(ci_tag=None):
-    if not ci_tag:
-        return "qa"
-    log.debug("getting build type for tag: \"%s\"", ci_tag)
-    if re.match(r'v\d+\.\d+\.\d+rc\d+$', ci_tag):
-        return 'rc'
-    elif re.match(r'v\d+\.\d+\.\d+$', ci_tag):
-        return 'release'
-    return 'qa'
+def _check_and_set(d: dict, key: str, value: str):
+    try:
+        d[key]
+    except KeyError:
+        raise Exception(f"{key} var does not exist in {build_info_mod.__file__}")
+    d[key] = value
 
 
 def main():
-    root_dir = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
-    build_type_path = os.path.join(root_dir, 'lbry', 'build_type.py')
-    log.debug("configuring build type file: %s", build_type_path)
-    commit_hash = os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT'))
+    build_info = {item: build_info_mod.__dict__[item] for item in dir(build_info_mod) if not item.startswith("__")}
+
+    commit_hash = os.getenv('DOCKER_COMMIT', os.getenv('CI_COMMIT_SHA', os.getenv('TRAVIS_COMMIT')))
     if commit_hash is None:
         raise ValueError("Commit hash not found in env vars")
-    commit_hash = commit_hash[:6]
-    build_type = get_build_type(os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG')))
-    log.debug("setting build type=%s, build commit=%s", build_type, commit_hash)
-    with open(build_type_path, 'w') as f:
-        f.write(f"BUILD = \"{build_type}\"\nCOMMIT_HASH = \"{commit_hash}\"\n")
+    _check_and_set(build_info, "COMMIT_HASH", commit_hash[:6])
+
+    docker_tag = os.getenv('DOCKER_TAG')
+    if docker_tag:
+        _check_and_set(build_info, "DOCKER_TAG", docker_tag)
+        _check_and_set(build_info, "BUILD", "docker")
+    else:
+        ci_tag = os.getenv('CI_COMMIT_TAG', os.getenv('TRAVIS_TAG'))
+        _check_and_set(build_info, "BUILD", "release" if re.match(r'v\d+\.\d+\.\d+$', str(ci_tag)) else "qa")
+
+    log.debug("build info: %s", ", ".join([f"{k}={v}" for k, v in build_info.items()]))
+    with open(build_info_mod.__file__, 'w') as f:
+        f.write("\n".join([f"{k} = \"{v}\"" for k, v in build_info.items()]) + "\n")
 
 
 if __name__ == '__main__':

lbry/build_info.py

@@ -1,3 +1,4 @@
 # don't touch this. CI server changes this during build/deployment
 BUILD = "dev"
 COMMIT_HASH = "none"
+DOCKER_TAG = "none"

lbry/extras/system_info.py

@@ -2,7 +2,7 @@ import platform
 import os
 import logging.handlers
 
-from lbry import build_type, __version__ as lbrynet_version
+from lbry import build_info, __version__ as lbrynet_version
 
 log = logging.getLogger(__name__)
 
@@ -19,7 +19,7 @@ def get_platform() -> dict:
         "os_system": os_system,
         "lbrynet_version": lbrynet_version,
         "version": lbrynet_version,
-        "build": build_type.BUILD,  # CI server sets this during build step
+        "build": build_info.BUILD,  # CI server sets this during build step
     }
     if d["os_system"] == "Linux":
         import distro  # pylint: disable=import-outside-toplevel

lbry/wallet/server/prometheus.py

@@ -1,14 +1,20 @@
 from aiohttp import web
 from prometheus_client import Counter, Info, generate_latest as prom_generate_latest
-from lbry.wallet.server import util
 from lbry import __version__ as version
-from lbry.build_type import BUILD, COMMIT_HASH
+from lbry.build_info import BUILD, COMMIT_HASH
+from lbry.wallet.server import util
+import lbry.wallet.server.version as wallet_server_version
 
 NAMESPACE = "wallet_server"
 
-VERSION_INFO = Info('build_info', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
-VERSION_INFO.info({'version': version, 'build': BUILD, "commit": COMMIT_HASH})
-REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE)
+VERSION_INFO = Info('build', 'Wallet server build info (e.g. version, commit hash)', namespace=NAMESPACE)
+VERSION_INFO.info({
+    'build': BUILD,
+    "commit": COMMIT_HASH,
+    'version': version,
+    "min_version": util.version_string(wallet_server_version.PROTOCOL_MIN),
+})
+REQUESTS_COUNT = Counter("requests_count", "Number of requests received", namespace=NAMESPACE, labelnames=("method",))
 
 
 class PrometheusServer:

lbry/wallet/server/session.py

@@ -28,6 +28,7 @@ from lbry.wallet.server.db import reader
 from lbry.wallet.server.websocket import AdminWebSocket
 from lbry.wallet.server.metrics import ServerLoadData, APICallMetrics
 from lbry.wallet.server.prometheus import REQUESTS_COUNT
+import lbry.wallet.server.version as VERSION
 
 from lbry.wallet.rpc import (
     RPCSession, JSONRPCAutoDetect, JSONRPCConnection,
@@ -708,7 +709,7 @@ class SessionBase(RPCSession):
         """Handle an incoming request.  ElectrumX doesn't receive
         notifications from client sessions.
         """
-        REQUESTS_COUNT.inc()
+        REQUESTS_COUNT.labels(method=request.method).inc()
         if isinstance(request, Request):
             handler = self.request_handlers.get(request.method)
         else:
@@ -778,8 +779,8 @@ class LBRYSessionManager(SessionManager):
 class LBRYElectrumX(SessionBase):
     """A TCP server that handles incoming Electrum connections."""
 
-    PROTOCOL_MIN = (0, 54, 0)
-    PROTOCOL_MAX = (0, 99, 0)
+    PROTOCOL_MIN = VERSION.PROTOCOL_MIN
+    PROTOCOL_MAX = VERSION.PROTOCOL_MAX
     max_errors = math.inf  # don't disconnect people for errors! let them happen...
     session_mgr: LBRYSessionManager
     version = lbry.__version__

lbry/wallet/server/version.py

@@ -0,0 +1,3 @@
+# need this to avoid circular import
+PROTOCOL_MIN = (0, 54, 0)
+PROTOCOL_MAX = (0, 99, 0)