Added ansible and decoder

Added ansible playbook and decoder into this repo.
This commit is contained in:
Fillerino 2017-09-05 20:11:29 +02:00
parent 5c929f343c
commit 229e4fa8c8
29 changed files with 603 additions and 1 deletions

2
.gitignore vendored
View file

@ -2,3 +2,5 @@ node_modules
.DS_Store .DS_Store
/dist /dist
npm-debug.log npm-debug.log
*.retry
.vscode

1
ansible-lighthouse/hosts Normal file
View file

@ -0,0 +1 @@
[lighthouse]

View file

@ -0,0 +1,19 @@
- name: Lighthouse - Download lbrycrd
hosts: lighthouse
tasks:
- raw: apt-get update && apt-get install -y wget unzip screen
- raw: cd ~ && wget https://github.com/lbryio/lbrycrd/releases/download/v0.12.0.5/lbrycrd-linux.zip && unzip lbrycrd-linux.zip -d lbrycrd
- raw: cd ~/lbrycrd && screen -S lbrycrd -d -m ./lbrycrdd -server -txindex -reindex -rpcuser=lbry -rpcpassword=lbry && sleep 2
- name: Lighthouse - Install elasticsearch, download and install lighthouse
hosts: lighthouse
roles:
- { role: elasticsearch, es_instance_name: "node1", es_heap_size: "256m" }
- { role: geerlingguy.nodejs, nodejs_version: "8.x" }
- { role: yarn }
- { role: bobbyrenwick.pip }
tasks:
- git: repo=https://github.com/lbryio/lighthouse.git dest=~/lighthouse
- raw: cd ~/lighthouse/decoder && pip install -r requirements.txt && screen -S decoder -d -m python decoder.py && sleep 2
- raw: cd ~/lighthouse && yarn install --production=false && yarn global add forever
- raw: forever start -c "npm run prod" ~/lighthouse

View file

@ -0,0 +1 @@
.idea

View file

@ -0,0 +1,19 @@
ansible-pip
===========
An ansible role that ensures pip is installed at the version you specify.
It uses `get-pip.py` to install pip.
See role in [ansible-galaxy](https://galaxy.ansible.com/bobbyrenwick/pip/)
Role Variables
--------------
- `pip_download_dest` specifies where `get-pip.py` should be downloaded to. Defaults to `/tmp`
- `pip_version` specifies which version of pip you want to install.
- Defaults to `None`, to install the latest version (If pip is installed, upgrades to the latest pip version available)
- Can be set to a specific version, e.g. `"6.0.8"` to force installation of that version.
- Can be set explicitly to `"LATEST"` or `"latest"` to force upgrade to the latest available version (same behaviour as `None`).
- `python` specifies what Python executable to use. Defaults to `python`.
- `pip` specifies what pip executable to check and use. Defaults to `pip`.
- `pip_proxy` specifies a HTTP proxy, if you have to use it (e.g. behind firewall). Default is `` to skip it.

View file

@ -0,0 +1,7 @@
---
pip_download_dest: /tmp
pip_version:
python: python
pip: pip
pip_proxy: ''

View file

@ -0,0 +1 @@
{install_date: 'Tue Sep 5 17:35:23 2017', version: v2.1.1}

View file

@ -0,0 +1,108 @@
---
galaxy_info:
author: bobbyrenwick
description: An ansible role that ensures pip is installed at the version you specify.
license: WTFPL
min_ansible_version: 1.9
#
# Below are all platforms currently available. Just uncomment
# the ones that apply to your role. If you don't see your
# platform on this list, let us know and we'll get it added!
#
platforms:
- name: EL
versions:
- all
- 5
- 6
- name: GenericUNIX
versions:
- all
- any
- name: Fedora
versions:
- all
- 16
- 17
- 18
- 19
- 20
- name: opensuse
versions:
- all
- 12.1
- 12.2
- 12.3
- 13.1
- 13.2
- name: GenericBSD
versions:
- all
- any
- name: FreeBSD
versions:
- all
- 8.0
- 8.1
- 8.2
- 8.3
- 8.4
- 9.0
- 9.1
- 9.1
- 9.2
- name: Ubuntu
versions:
- all
- lucid
- maverick
- natty
- oneiric
- precise
- quantal
- raring
- saucy
- trusty
- name: SLES
versions:
- all
- 10SP3
- 10SP4
- 11
- 11SP1
- 11SP2
- 11SP3
- name: GenericLinux
versions:
- all
- any
- name: Debian
versions:
- all
- etch
- lenny
- squeeze
- wheezy
# Below are all categories currently available. Just as with
# the platforms above, uncomment those that apply to your role.
#
categories:
#- cloud
#- cloud:ec2
#- cloud:gce
#- cloud:rax
#- database
#- database:nosql
#- database:sql
#- development
#- monitoring
#- networking
#- packaging
- system
#- web
dependencies: []
# List your role dependencies here, one per line. Only
# dependencies available via galaxy should be listed here.
# Be sure to remove the '[]' above if you add dependencies
# to this list.

View file

@ -0,0 +1,44 @@
---
# Causes an error if we try and which something that doesn't exist so use this
# as a workaround.
- name: Check to see if pip is already installed.
command: "{{ pip }} --version"
ignore_errors: true
changed_when: false # read-only task
check_mode: no
register: pip_is_installed
- name: Download pip.
get_url: url=https://bootstrap.pypa.io/get-pip.py dest={{ pip_download_dest }}
when: pip_is_installed.rc != 0
- name: Install pip.
command: "{{ python }} {{ pip_download_dest }}/get-pip.py{{ ' --proxy=' + pip_proxy if pip_proxy != '' else '' }}"
become: yes
when: pip_is_installed.rc != 0
- name: Delete get-pip.py.
file: state=absent path={{ pip_download_dest }}/get-pip.py
when: pip_is_installed.rc != 0
# $ pip --version
# pip 1.5.2 from /usr/local/lib/python2.7/dist-packages (python 2.7)
- name: Check to see if pip is installed at the correct version.
shell: "{{ pip }} --version | awk '{print $2}'"
register: pip_installed_version
changed_when: false
check_mode: no
when: pip_version or (pip_version | lower) != "latest"
- name: Install required version of pip.
command: "{{ pip }} install pip=={{ pip_version }}"
become: yes
when: pip_version and pip_installed_version.stdout != pip_version and (pip_version | lower) != "latest"
- name: Upgrade to latest version of pip.
command: "{{ pip }} {{ '--proxy=' + pip_proxy + ' ' if pip_proxy != '' else '' }}install -U pip"
register: pip_latest_output
become: yes
changed_when: pip_latest_output.stdout.find('Requirement already up-to-date') == -1
when: not pip_version or (pip_version | lower) == "latest"

@ -0,0 +1 @@
Subproject commit ea5a373155e96dfc4941b2bec3920e7f76eb28de

View file

@ -0,0 +1,2 @@
*.retry
tests/test.sh

View file

@ -0,0 +1,41 @@
---
services: docker
env:
# Defaults.
- distro: centos7
- distro: centos6
- distro: ubuntu1604
- distro: ubuntu1404
- distro: debian9
- distro: debian8
# Latest release.
- distro: centos7
playbook: test-latest.yml
- distro: ubuntu1604
playbook: test-latest.yml
script:
# Configure test script so we can run extra tests after playbook is run.
- export container_id=$(date +%s)
- export cleanup=false
# Download test shim.
- wget -O ${PWD}/tests/test.sh https://gist.githubusercontent.com/geerlingguy/73ef1e5ee45d8694570f334be385e181/raw/
- chmod +x ${PWD}/tests/test.sh
# Run tests.
- ${PWD}/tests/test.sh
# Ensure Node.js is installed.
- 'docker exec --tty ${container_id} env TERM=xterm which node'
- 'docker exec --tty ${container_id} env TERM=xterm node -v'
# Ensure npm packages are installed globally.
- 'docker exec --tty ${container_id} env TERM=xterm bash --login -c "npm list -g --depth=0 jslint"'
- 'docker exec --tty ${container_id} env TERM=xterm bash --login -c "npm list -g --depth=0 node-sass"'
- 'docker exec --tty ${container_id} env TERM=xterm bash --login -c "npm list -g --depth=0 yo"'
notifications:
webhooks: https://galaxy.ansible.com/api/v1/notifications/

View file

@ -0,0 +1,20 @@
The MIT License (MIT)
Copyright (c) 2017 Jeff Geerling
Permission is hereby granted, free of charge, to any person obtaining a copy of
this software and associated documentation files (the "Software"), to deal in
the Software without restriction, including without limitation the rights to
use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
the Software, and to permit persons to whom the Software is furnished to do so,
subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

View file

@ -0,0 +1,73 @@
# Ansible Role: Node.js
[![Build Status](https://travis-ci.org/geerlingguy/ansible-role-nodejs.svg?branch=master)](https://travis-ci.org/geerlingguy/ansible-role-nodejs)
Installs Node.js on RHEL/CentOS or Debian/Ubuntu.
## Requirements
Requires the EPEL repository on RedHat/CentOS (you can install it by simply adding the `geerlingguy.repo-epel` role to your playbook).
## Role Variables
Available variables are listed below, along with default values (see `defaults/main.yml`):
nodejs_version: "6.x"
The Node.js version to install. "6.x" is the default and works on most supported OSes. Other versions such as "0.12", "4.x", "5.x", "6.x", etc. should work on the latest versions of Debian/Ubuntu and RHEL/CentOS.
nodejs_install_npm_user: "{{ ansible_ssh_user }}"
The user for whom the npm packages will be installed can be set here, this defaults to `ansible_user`.
npm_config_prefix: "/usr/local/lib/npm"
The global installation directory. This should be writeable by the `nodejs_install_npm_user`.
npm_config_unsafe_perm: "false"
Set to true to suppress the UID/GID switching when running package scripts. If set explicitly to false, then installing as a non-root user will fail.
nodejs_npm_global_packages: []
A list of npm packages with a `name` and (optional) `version` to be installed globally. For example:
nodejs_npm_global_packages:
# Install a specific version of a package.
- name: jslint
version: 0.9.3
# Install the latest stable release of a package.
- name: node-sass
# This shorthand syntax also works (same as previous example).
- node-sass
<!-- code block separator -->
nodejs_package_json_path: ""
Set a path pointing to a particular `package.json` (e.g. `"/var/www/app/package.json"`). This will install all of the defined packages globally using Ansible's `npm` module.
## Dependencies
None.
## Example Playbook
- hosts: utility
vars_files:
- vars/main.yml
roles:
- geerlingguy.nodejs
*Inside `vars/main.yml`*:
nodejs_npm_global_packages:
- name: jslint
- name: node-sass
## License
MIT / BSD
## Author Information
This role was created in 2014 by [Jeff Geerling](https://www.jeffgeerling.com/), author of [Ansible for DevOps](https://www.ansiblefordevops.com/).

View file

@ -0,0 +1,26 @@
---
# Set the version of Node.js to install ("0.12", "4.x", "5.x", "6.x", "8.x").
# Version numbers from Nodesource: https://github.com/nodesource/distributions
nodejs_version: "6.x"
# The user for whom the npm packages will be installed.
# nodejs_install_npm_user: username
# The directory for global installations.
npm_config_prefix: "/usr/local/lib/npm"
# Set to true to suppress the UID/GID switching when running package scripts. If set explicitly to false, then installing as a non-root user will fail.
npm_config_unsafe_perm: "false"
# Define a list of global packages to be installed with NPM.
nodejs_npm_global_packages: []
# # Install a specific version of a package.
# - name: jslint
# version: 0.9.3
# # Install the latest stable release of a package.
# - name: node-sass
# # This shorthand syntax also works (same as previous example).
# - node-sass
# The path of a package.json file used to install packages globally.
nodejs_package_json_path: ""

View file

@ -0,0 +1 @@
{install_date: 'Tue Sep 5 16:35:39 2017', version: 4.1.2}

View file

@ -0,0 +1,24 @@
---
dependencies: []
galaxy_info:
author: geerlingguy
description: Node.js installation for Linux
company: "Midwestern Mac, LLC"
license: "license (BSD, MIT)"
min_ansible_version: 1.9
platforms:
- name: EL
versions:
- 6
- 7
- name: Debian
versions:
- all
- name: Ubuntu
versions:
- trusty
- xenial
galaxy_tags:
- development
- web

View file

@ -0,0 +1,41 @@
---
- include: setup-RedHat.yml
when: ansible_os_family == 'RedHat'
- include: setup-Debian.yml
when: ansible_os_family == 'Debian'
- name: Define nodejs_install_npm_user
set_fact:
nodejs_install_npm_user: "{{ ansible_user | default(lookup('env', 'USER')) }}"
when: nodejs_install_npm_user is not defined
- name: Create npm global directory
file:
path: "{{ npm_config_prefix }}"
owner: "{{ nodejs_install_npm_user }}"
group: "{{ nodejs_install_npm_user }}"
state: directory
- name: Add npm_config_prefix bin directory to global $PATH.
template:
src: npm.sh.j2
dest: /etc/profile.d/npm.sh
mode: 0644
- name: Ensure npm global packages are installed.
npm:
name: "{{ item.name | default(item) }}"
version: "{{ item.version | default('latest') }}"
global: yes
state: latest
environment:
NPM_CONFIG_PREFIX: "{{ npm_config_prefix }}"
NODE_PATH: "{{ npm_config_prefix }}/lib/node_modules"
NPM_CONFIG_UNSAFE_PERM: "{{ npm_config_unsafe_perm }}"
with_items: "{{ nodejs_npm_global_packages }}"
- name: Install packages defined in a given package.json.
npm:
path: "{{ nodejs_package_json_path }}"
when: nodejs_package_json_path is defined and nodejs_package_json_path

View file

@ -0,0 +1,25 @@
---
- name: Ensure apt-transport-https is installed.
apt: name=apt-transport-https state=present
- name: Add Nodesource apt key.
apt_key:
url: https://keyserver.ubuntu.com/pks/lookup?op=get&fingerprint=on&search=0x1655A0AB68576280
id: "68576280"
state: present
- name: Add NodeSource repositories for Node.js.
apt_repository:
repo: "{{ item }}"
state: present
with_items:
- "deb https://deb.nodesource.com/node_{{ nodejs_version }} {{ ansible_distribution_release }} main"
- "deb-src https://deb.nodesource.com/node_{{ nodejs_version }} {{ ansible_distribution_release }} main"
register: node_repo
- name: Update apt cache if repo was added.
apt: update_cache=yes
when: node_repo.changed
- name: Ensure Node.js and npm are installed.
apt: "name=nodejs={{ nodejs_version|regex_replace('x', '') }}* state=present"

View file

@ -0,0 +1,37 @@
---
- name: Set up the Nodesource RPM directory for Node.js > 0.10.
set_fact:
nodejs_rhel_rpm_dir: "pub_{{ nodejs_version }}"
when: nodejs_version != '0.10'
- name: Set up the Nodesource RPM variable for Node.js == 0.10.
set_fact:
nodejs_rhel_rpm_dir: "pub"
when: nodejs_version == '0.10'
- name: Import Nodesource RPM key (CentOS < 7).
rpm_key:
key: http://rpm.nodesource.com/pub/el/NODESOURCE-GPG-SIGNING-KEY-EL
state: present
when: ansible_distribution_major_version|int < 7
- name: Import Nodesource RPM key (CentOS 7+)..
rpm_key:
key: https://rpm.nodesource.com/pub/el/NODESOURCE-GPG-SIGNING-KEY-EL
state: present
when: ansible_distribution_major_version|int >= 7
- name: Add Nodesource repositories for Node.js (CentOS < 7).
yum:
name: "http://rpm.nodesource.com/{{ nodejs_rhel_rpm_dir }}/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm"
state: present
when: ansible_distribution_major_version|int < 7
- name: Add Nodesource repositories for Node.js (CentOS 7+).
yum:
name: "https://rpm.nodesource.com/{{ nodejs_rhel_rpm_dir }}/el/{{ ansible_distribution_major_version }}/{{ ansible_architecture }}/nodesource-release-el{{ ansible_distribution_major_version }}-1.noarch.rpm"
state: present
when: ansible_distribution_major_version|int >= 7
- name: Ensure Node.js and npm are installed.
yum: "name=nodejs-{{ nodejs_version[0] }}.* state=present enablerepo='epel,nodesource'"

View file

@ -0,0 +1,3 @@
export PATH={{ npm_config_prefix }}/bin:$PATH
export NPM_CONFIG_PREFIX={{ npm_config_prefix }}
export NODE_PATH=$NODE_PATH:{{ npm_config_prefix }}/lib/node_modules

View file

@ -0,0 +1,11 @@
# Ansible Role tests
To run the test playbook(s) in this directory:
1. Install and start Docker.
1. Download the test shim (see .travis.yml file for the URL) into `tests/test.sh`:
- `wget -O tests/test.sh https://gist.githubusercontent.com/geerlingguy/73ef1e5ee45d8694570f334be385e181/raw/`
1. Make the test shim executable: `chmod +x tests/test.sh`.
1. Run (from the role root directory) `distro=[distro] playbook=[playbook] ./tests/test.sh`
If you don't want the container to be automatically deleted after the test playbook is run, add the following environment variables: `cleanup=false container_id=$(date +%s)`

View file

@ -0,0 +1,21 @@
---
- hosts: all
vars:
nodejs_version: "8.x"
nodejs_install_npm_user: root
npm_config_prefix: /root/.npm-global
npm_config_unsafe_perm: "true"
nodejs_npm_global_packages:
- node-sass
- name: jslint
version: 0.9.6
- name: yo
pre_tasks:
- name: Update apt cache.
apt: update_cache=yes cache_valid_time=600
when: ansible_os_family == 'Debian'
roles:
- role_under_test

View file

@ -0,0 +1,20 @@
---
- hosts: all
vars:
nodejs_install_npm_user: root
npm_config_prefix: /root/.npm-global
npm_config_unsafe_perm: "true"
nodejs_npm_global_packages:
- node-sass
- name: jslint
version: 0.9.6
- name: yo
pre_tasks:
- name: Update apt cache.
apt: update_cache=yes cache_valid_time=600
when: ansible_os_family == 'Debian'
roles:
- role_under_test

@ -0,0 +1 @@
Subproject commit 65b7736cf0b8678fd1f23909109ed54bd181f8c7

1
decoder/config.json Normal file
View file

@ -0,0 +1 @@
{"rpc_user": "lbry", "rpc_password": "lbry", "rpc_port": 9245, "rpc_url": "127.0.0.1"}

49
decoder/decoder.py Normal file
View file

@ -0,0 +1,49 @@
import json
from bitcoinrpc.authproxy import AuthServiceProxy
from lbryschema.decode import smart_decode
from flask import Flask, url_for
app = Flask(__name__)
def get_lbrycrdd_connection_details():
with open('config.json', 'r') as f:
config = json.load(f)
rpc_user = config['rpc_user']
rpc_pass = config['rpc_password']
rpc_port = config['rpc_port']
rpc_url = config['rpc_url']
return "http://%s:%s@%s:%i" % (rpc_user, rpc_pass, rpc_url, rpc_port)
@app.errorhandler(500)
def internal_error(error):
return 'error when decoding claims'
@app.route('/claim_decode/<txid>/<nout>')
def api_decode(txid, nout):
connection_string = get_lbrycrdd_connection_details()
rpc = AuthServiceProxy(connection_string)
result = rpc.getclaimsfortx(txid)
claim = None
for claim_out in result:
if claim_out['nOut'] == int(nout):
claim = claim_out
break
if claim:
converted = "".join([chr(ord(i)) for i in claim['value']])
decoded = smart_decode(converted) # Decode the claims and dump them back to logstash plugin
claim['value'] = decoded.claim_dict
return json.dumps(claim)
@app.route('/claim_decodeinv/<claimid>')
def api_decodebyclaim(claimid):
connection_string = get_lbrycrdd_connection_details()
rpc = AuthServiceProxy(connection_string)
claim = rpc.getvalueforname(claimid)
if claim:
converted = "".join([chr(ord(i)) for i in claim['value']])
decoded = smart_decode(converted) # Decode the claims and dump them back to logstash plugin
claim['value'] = decoded.claim_dict
return json.dumps(claim)
if __name__ == '__main__':
app.run(host='127.0.0.1')

3
decoder/requirements.txt Normal file
View file

@ -0,0 +1,3 @@
git+https://github.com/lbryio/lbryschema.git#egg=lbryschema
python-bitcoinrpc==0.1
flask

View file

@ -7,7 +7,7 @@ import elasticsearch from 'elasticsearch';
const loggerStream = winstonStream(winston, 'info'); const loggerStream = winstonStream(winston, 'info');
const eclient = new elasticsearch.Client({ const eclient = new elasticsearch.Client({
host: 'http://elastic:changeme@localhost:9200', host: 'http://localhost:9200',
log : { log : {
level : 'info', level : 'info',
type : 'stream', type : 'stream',