Merge pull request #461 from mrd0ll4r/travis-e2e

travis: add e2e tests with redis storage

.travis.yml

@@ -2,61 +2,32 @@ language: go
 go:
   - 1.13.x
 sudo: false
+services:
+  - redis-server
 matrix:
   include:
-  # Using vendored dependencies
-  - install:
-    - go get -u golang.org/x/lint/golint
-    - go get -u golang.org/x/tools/cmd/...
-    script:
-    - go mod download
-    - go test -v -race $(go list ./...)
-    - go vet $(go list ./...)
-    - diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "")
-    - (for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1;  done)
-    - go install github.com/chihaya/chihaya/cmd/chihaya
-    - chihaya --config=example_config.yaml --debug&
-    - pid=$!
-    - sleep 2 # wait for chihaya to start up (gross)
-    - chihaya e2e --debug
-    - kill $pid
-  # Using HEAD of dependencies
-  - install:
-    - go get -t -u ./...
-    - go get -u golang.org/x/lint/golint
-    - go get -u golang.org/x/tools/cmd/...
-    script:
-    - go test -v -race $(go list ./...)
-    - go vet $(go list ./...)
-    - diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "")
-    - (for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1;  done)
-    - go install github.com/chihaya/chihaya/cmd/chihaya
-    - chihaya --config=example_config.yaml --debug&
-    - pid=$!
-    - sleep 2 # wait for chihaya to start up (gross)
-    - chihaya e2e --debug
-    - kill $pid
+    # Using vendored dependencies
+    - install:
+        - ./dist/travis/install_reproducible.sh
+      script:
+        - ./dist/travis/check.sh
+
+    # Using HEAD of dependencies
+    - install:
+        - ./dist/travis/install_tip.sh
+      script:
+        - ./dist/travis/check.sh
+
   allow_failures:
-  # Using HEAD of dependencies
-  - install:
-    - go get -t -u ./...
-    - go get -u golang.org/x/lint/golint
-    - go get -u golang.org/x/tools/cmd/...
-    script:
-    - go test -v -race $(go list ./...)
-    - go vet $(go list ./...)
-    - diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "")
-    - (for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1;  done)
-    - go install github.com/chihaya/chihaya/cmd/chihaya
-    - chihaya --config=example_config.yaml --debug&
-    - pid=$!
-    - sleep 2 # wait for chihaya to start up (gross)
-    - chihaya e2e --debug
-    - kill $pid
+    # Using HEAD of dependencies
+    - install:
+        - ./dist/travis/install_tip.sh
+      script:
+        - ./dist/travis/check.sh
 notifications:
   irc:
     channels:
-    - irc.freenode.net#chihaya
+      - irc.freenode.net#chihaya
     use_notice: true
     skip_join: true
     on_success: always

Dockerfile

@@ -9,7 +9,7 @@ WORKDIR /go/src/github.com/chihaya/chihaya
 COPY . /go/src/github.com/chihaya/chihaya
 
 # Install our golang dependencies and compile our binary.
-RUN CGO_ENABLED=0 GO111MODULE=on go install github.com/chihaya/chihaya/cmd/...
+RUN CGO_ENABLED=0 go install github.com/chihaya/chihaya/cmd/...
 
 FROM alpine:latest
 RUN apk add --no-cache ca-certificates

README.md

@@ -61,7 +61,7 @@ In order to compile the project, the [latest stable version of Go] and knowledge
 ```sh
 $ git clone git@github.com:chihaya/chihaya.git
 $ cd chihaya
-$ GO111MODULE=on go build ./cmd/chihaya
+$ go build ./cmd/chihaya
 $ ./chihaya --help
 ```
 
@@ -84,6 +84,19 @@ Removing `-bench` will just run unit tests.
 $ go test -bench $(go list ./...)
 ```
 
+The Chihaya executable contains a command to end-to-end test a BitTorrent tracker.
+See
+
+```sh
+$ chihaya --help
+``` 
+
+### Configuration
+
+Configuration of Chihaya is done via one YAML configuration file.
+The `dist/` directory contains an example configuration file.
+Files and directories under `docs/` contain detailed information about configuring middleware, storage implementations, architecture etc.
+
 ## Related projects
 
 - [BitTorrent.org](https://github.com/bittorrent/bittorrent.org): a static website containing the BitTorrent spec and all BEPs

dist/example_config.yaml

@@ -129,14 +129,15 @@ chihaya:
   #     # This balances between
   #     # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value)
   #     # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value).
-  #     gc_interval: 14m
+  #     gc_interval: 3m
 
-  #     # The frequency which metrics are pushed into a local Prometheus endpoint.
+  #     # The interval at which metrics about the number of infohashes and peers
+  #     # are collected and posted to Prometheus.
   #     prometheus_reporting_interval: 1s
 
   #     # The amount of time until a peer is considered stale.
   #     # To avoid churn, keep this slightly larger than `announce_interval`
-  #     peer_lifetime: 16m
+  #     peer_lifetime: 31m
 
   #     # The address of redis storage.
   #     redis_broker: "redis://pwd@127.0.0.1:6379/0"

dist/travis/check.sh

@@ -0,0 +1,22 @@
+#!/bin/bash
+set -ex
+
+go test -v -race $(go list ./...)
+go vet $(go list ./...)
+diff <(goimports -local github.com/chihaya/chihaya -d $(find . -type f -name '*.go' -not -path "./vendor/*")) <(printf "")
+(for d in $(go list ./...); do diff <(golint $d) <(printf "") || exit 1;  done)
+go install github.com/chihaya/chihaya/cmd/chihaya
+
+# Run e2e test with example config.
+chihaya --config=./dist/travis/config_memory.yaml --debug&
+pid=$!
+sleep 2 # wait for Chihaya to start up (gross)
+chihaya e2e --debug
+kill $pid
+
+# Run e2e test with redis.
+chihaya --config=./dist/travis/config_redis.yaml --debug&
+pid=$!
+sleep 2 # wait for Chihaya to start up (gross)
+chihaya e2e --debug
+kill $pid

dist/travis/config_memory.yaml

@@ -0,0 +1,42 @@
+# This config file is used by travis for end-to-end testing.
+# See example_config.yaml for a commented, more complete configuration!
+
+chihaya:
+  announce_interval: 30m
+  min_announce_interval: 15m
+  prometheus_addr: "0.0.0.0:6880"
+
+  http:
+    addr: "0.0.0.0:6969"
+    https_addr: ""
+    tls_cert_path: ""
+    tls_key_path: ""
+    read_timeout: 5s
+    write_timeout: 5s
+    enable_keepalive: false
+    idle_timeout: 30s
+    enable_request_timing: false
+    enable_legacy_php_urls: false
+    allow_ip_spoofing: false
+    real_ip_header: "x-real-ip"
+    max_numwant: 100
+    default_numwant: 50
+    max_scrape_infohashes: 50
+
+  udp:
+    addr: "0.0.0.0:6969"
+    max_clock_skew: 10s
+    private_key: "paste a random string here that will be used to hmac connection IDs"
+    enable_request_timing: false
+    allow_ip_spoofing: false
+    max_numwant: 100
+    default_numwant: 50
+    max_scrape_infohashes: 50
+
+  storage:
+    name: memory
+    config:
+      gc_interval: 3m
+      peer_lifetime: 31m
+      shard_count: 1024
+      prometheus_reporting_interval: 1s

dist/travis/config_redis.yaml

@@ -0,0 +1,45 @@
+# This config file is used by travis for end-to-end testing.
+# See example_config.yaml for a commented, more complete configuration!
+
+chihaya:
+  announce_interval: 30m
+  min_announce_interval: 15m
+  prometheus_addr: "0.0.0.0:6880"
+
+  http:
+    addr: "0.0.0.0:6969"
+    https_addr: ""
+    tls_cert_path: ""
+    tls_key_path: ""
+    read_timeout: 5s
+    write_timeout: 5s
+    enable_keepalive: false
+    idle_timeout: 30s
+    enable_request_timing: false
+    enable_legacy_php_urls: false
+    allow_ip_spoofing: false
+    real_ip_header: "x-real-ip"
+    max_numwant: 100
+    default_numwant: 50
+    max_scrape_infohashes: 50
+
+  udp:
+    addr: "0.0.0.0:6969"
+    max_clock_skew: 10s
+    private_key: "paste a random string here that will be used to hmac connection IDs"
+    enable_request_timing: false
+    allow_ip_spoofing: false
+    max_numwant: 100
+    default_numwant: 50
+    max_scrape_infohashes: 50
+
+  storage:
+    name: redis
+    config:
+      gc_interval: 3m
+      peer_lifetime: 31m
+      prometheus_reporting_interval: 1s
+      redis_broker: "redis://127.0.0.1:6379/0"
+      redis_read_timeout: 15s
+      redis_write_timeout: 15s
+      redis_connect_timeout: 15s

dist/travis/install_reproducible.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+
+# Install golint and go vet.
+go get -u golang.org/x/lint/golint
+go get -u golang.org/x/tools/cmd/...
+
+go mod download

dist/travis/install_tip.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+set -ex
+
+# Install golint and go vet.
+go get -u golang.org/x/lint/golint
+go get -u golang.org/x/tools/cmd/...
+
+go get -t -u ./...

docs/storage/redis.md

@@ -1,12 +1,20 @@
 # Redis Storage
 
-This storage system separates chihaya server from storage service, chihaya server achieves HA by storing all peer data in Redis, the storage service gets HA by creating cluster. If one chihaya node goes down, peer data will still be available in Redis service.
+This storage implementation separates Chihaya from its storage service.
+Chihaya achieves HA by storing all peer data in Redis.
+Multiple instances of Chihaya can use the same redis instance concurrently.
+The storage service can get HA by clustering.
+If one instance of Chihaya goes down, peer data will still be available in Redis.
 
-The HA of storage service is not considered here, it's another topic. In case Redis service is a single node, peer data will be unavailable if the node is down. So you should setup a Redis cluster for chihaya server in production.
+The HA of storage service is not considered here.
+In case Redis runs as a single node, peer data will be unavailable if the node is down.
+You should consider setting up a Redis cluster for Chihaya in production.
+
+This storage implementation is currently orders of magnitude slower than the in-memory implementation.
 
 ## Use Case
 
-When one chihaya instance is down, the Redis can continuily serve peer data through other chihaya instances.
+When one instance of Chihaya is down, other instances can continue serving peers from Redis.
 
 ## Configuration
 
@@ -16,14 +24,18 @@ chihaya:
     name: redis
     config:
       # The frequency which stale peers are removed.
-      gc_interval: 14m
+      # This balances between
+      # - collecting garbage more often, potentially using more CPU time, but potentially using less memory (lower value)
+      # - collecting garbage less frequently, saving CPU time, but keeping old peers long, thus using more memory (higher value).
+      gc_interval: 3m
 
-      # The frequency which metrics are pushed into a local Prometheus endpoint.
+      # The interval at which metrics about the number of infohashes and peers
+      # are collected and posted to Prometheus.
       prometheus_reporting_interval: 1s
 
       # The amount of time until a peer is considered stale.
       # To avoid churn, keep this slightly larger than `announce_interval`
-      peer_lifetime: 16m
+      peer_lifetime: 31m
 
       # The address of redis storage.
       redis_broker: "redis://pwd@127.0.0.1:6379/0"
@@ -40,11 +52,12 @@ chihaya:
 
 ## Implementation
 
-Seeders and Leechers for a particular InfoHash are stored with a redis hash structure, the infohash is used as hash key, peer key is field, last modified time is value.
+Seeders and Leechers for a particular InfoHash are stored within a redis hash.
+The InfoHash is used as key, _peer keys_ are the fields, last modified times are values.
+Peer keys are derived from peers and contain Peer ID, IP, and Port.
+All the InfoHashes (swarms) are also stored in a redis hash, with IP family as the key, infohash as field, and last modified time as value.
 
-All the InfoHashes (swarms) are also stored into redis hash, IP family is the key, infohash is field, last modified time is value.
-
-Here is an example
+Here is an example:
 
 ```
 - IPv4
@@ -61,9 +74,8 @@ Here is an example
 ```
 
 
-In this case, prometheus will record two swarms, three seeders and one leecher.
-
-So tree keys are used to record the count of swarms, seeders and leechers for each group (IPv4, IPv6).
+In this case, prometheus would record two swarms, three seeders, and one leecher.
+These three keys per address family are used to record the count of swarms, seeders, and leechers.
 
 ```
 - IPv4_infohash_count: 2
@@ -71,4 +83,5 @@ So tree keys are used to record the count of swarms, seeders and leechers for ea
 - IPv4_L_count: 1
 ```
 
-Note: IPv4_infohash_count has the different meaning with `memory` storage, it represents the number of infohashes reported by seeder.
+Note: IPv4_infohash_count has a different meaning compared to the `memory` storage:
+It represents the number of infohashes reported by seeder, meaning that infohashes without seeders are not counted.