Add dockerfile and docker-compose with es

This commit is contained in:
Jeffrey Picard 2021-06-04 12:38:17 -04:00
parent b557bf8237
commit 944496b076
7 changed files with 101 additions and 6 deletions

5
Dockerfile Normal file
View file

@ -0,0 +1,5 @@
FROM debian:10-slim
EXPOSE 50051
COPY ./hub /hub
ENTRYPOINT ["/hub", "serve"]

4
build.sh Executable file
View file

@ -0,0 +1,4 @@
#!/bin/bash
go build .
docker build . -t lbry/hub:latest

View file

@ -0,0 +1,38 @@
version: "3"
volumes:
es01:
services:
hub_server:
depends_on:
- es01
image: lbry/hub:latest
restart: always
ports:
- "50051:50051" # rpc port
environment:
#- TCP_PORT=50051 # should probably have these supported by the go server too
#- TCP_HOST=0.0.0.0
- ELASTIC_HOST=http://127.0.0.1
- ELASTIC_PORT=9200
es01:
image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1
container_name: es01
environment:
- node.name=es01
- discovery.type=single-node
- indices.query.bool.max_clause_count=4096
- bootstrap.memory_lock=true
- "ES_JAVA_OPTS=-Xms512m -Xmx512m" # no more than 32, remember to disable swap
#- "ES_JAVA_OPTS=-Xms8g -Xmx8g" # no more than 32, remember to disable swap
ulimits:
memlock:
soft: -1
hard: -1
volumes:
- es01:/usr/share/elasticsearch/data
ports:
- "9200:9200"
- "9300:9300"
#- 127.0.0.1:9200:9200

39
main.go
View file

@ -7,6 +7,7 @@ import (
"log" "log"
"net" "net"
"os" "os"
"strings"
"time" "time"
"github.com/akamensky/argparse" "github.com/akamensky/argparse"
@ -21,6 +22,8 @@ const (
defaultPort = "50051" defaultPort = "50051"
defaultRPCUser = "rpcuser" defaultRPCUser = "rpcuser"
defaultRPCPassword = "rpcpassword" defaultRPCPassword = "rpcpassword"
defaultEsHost = "http://localhost"
defaultEsPort = "9200"
) )
type loginCreds struct { type loginCreds struct {
@ -39,6 +42,21 @@ func (c *loginCreds) RequireTransportSecurity() bool {
} }
func parseArgs(searchRequest *pb.SearchRequest) *server.Args { func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
getenvironment := func(data []string, getkeyval func(item string) (key, val string)) map[string]string {
items := make(map[string]string)
for _, item := range data {
key, val := getkeyval(item)
items[key] = val
}
return items
}
environment := getenvironment(os.Environ(), func(item string) (key, val string) {
splits := strings.Split(item, "=")
key = splits[0]
val = splits[1]
return
})
parser := argparse.NewParser("hub", "hub server and client") parser := argparse.NewParser("hub", "hub server and client")
serveCmd := parser.NewCommand("serve", "start the hub server") serveCmd := parser.NewCommand("serve", "start the hub server")
@ -63,7 +81,23 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
log.Fatalln(parser.Usage(err)) log.Fatalln(parser.Usage(err))
} }
args := &server.Args{Serve: false, Port: ":" + *port, User: *user, Pass: *pass}
args := &server.Args{
Serve: false,
Port: ":" + *port,
User: *user,
Pass: *pass,
EsHost: defaultEsHost,
EsPort: defaultEsPort,
}
if esHost, ok := environment["ELASTIC_HOST"]; ok {
args.EsHost = esHost
}
if esPort, ok := environment["ELASTIC_PORT"]; ok {
args.EsPort = esPort
}
/* /*
Verify no invalid argument combinations Verify no invalid argument combinations
@ -109,6 +143,9 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
func main() { func main() {
searchRequest := &pb.SearchRequest{} searchRequest := &pb.SearchRequest{}
//
//res := schema.ParseURL("@abc#1111")
//log.Println(res)
args := parseArgs(searchRequest) args := parseArgs(searchRequest)

View file

@ -7,6 +7,7 @@ import (
"regexp" "regexp"
"strconv" "strconv"
"strings" "strings"
"unicode/utf16"
) )
type PathSegment struct { type PathSegment struct {
@ -123,9 +124,16 @@ func ParseURL(url string) *URL {
} }
func createUrlRegex() *regexp.Regexp { func createUrlRegex() *regexp.Regexp {
//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+" d800 := []uint16{0xd800}
dfff := []uint16{0xdfff}
s1 := string(utf16.Decode(d800))
s2 := string(utf16.Decode(dfff))
log.Println(s1)
log.Println(s2)
//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\\u0000-\\u0020\\uD800-\\uDFFF\\uFFFE-\\uFFFF]+"
invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020" + s1 + "-" + s2 + "\uFFFE-\uFFFF]+"
//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020-\uFFFE-\uFFFF]+" //invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020-\uFFFE-\uFFFF]+"
invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "]+" //invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "]+"
named := func (name string, regex string) string { named := func (name string, regex string) string {
return "(?P<" + name + ">" + regex + ")" return "(?P<" + name + ">" + regex + ")"

View file

@ -341,7 +341,8 @@ func (s *Server) resolveUrl(ctx context.Context, rawUrl string) *urlResolution {
func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) { func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) {
var client *elastic.Client = nil var client *elastic.Client = nil
if s.EsClient == nil { if s.EsClient == nil {
tmpClient, err := elastic.NewClient(elastic.SetSniff(false)) esUrl := s.Args.EsHost + ":" + s.Args.EsPort
tmpClient, err := elastic.NewClient(elastic.SetURL(esUrl), elastic.SetSniff(false))
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -351,8 +352,8 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
client = s.EsClient client = s.EsClient
} }
res := s.resolveUrl(ctx, "@abc#111") //res := s.resolveUrl(ctx, "@abc#111")
log.Println(res) //log.Println(res)
claimTypes := map[string]int { claimTypes := map[string]int {
"stream": 1, "stream": 1,

View file

@ -24,6 +24,8 @@ type Args struct {
Port string Port string
User string User string
Pass string Pass string
EsHost string
EsPort string
} }
type AccessDeniedErr struct {} type AccessDeniedErr struct {}