From 944496b07618b766e73dc730a054d29a45f02fd8 Mon Sep 17 00:00:00 2001 From: Jeffrey Picard Date: Fri, 4 Jun 2021 12:38:17 -0400 Subject: [PATCH] Add dockerfile and docker-compose with es --- Dockerfile | 5 +++++ build.sh | 4 ++++ docker-compose-hub-server.yml | 38 ++++++++++++++++++++++++++++++++++ main.go | 39 ++++++++++++++++++++++++++++++++++- schema/url.go | 12 +++++++++-- server/search.go | 7 ++++--- server/server.go | 2 ++ 7 files changed, 101 insertions(+), 6 deletions(-) create mode 100644 Dockerfile create mode 100755 build.sh create mode 100644 docker-compose-hub-server.yml diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..c385cff --- /dev/null +++ b/Dockerfile @@ -0,0 +1,5 @@ +FROM debian:10-slim + +EXPOSE 50051 +COPY ./hub /hub +ENTRYPOINT ["/hub", "serve"] diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..8128a1f --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +go build . +docker build . -t lbry/hub:latest \ No newline at end of file diff --git a/docker-compose-hub-server.yml b/docker-compose-hub-server.yml new file mode 100644 index 0000000..7479e75 --- /dev/null +++ b/docker-compose-hub-server.yml @@ -0,0 +1,38 @@ +version: "3" + +volumes: + es01: + +services: + hub_server: + depends_on: + - es01 + image: lbry/hub:latest + restart: always + ports: + - "50051:50051" # rpc port + environment: + #- TCP_PORT=50051 # should probably have these supported by the go server too + #- TCP_HOST=0.0.0.0 + - ELASTIC_HOST=http://127.0.0.1 + - ELASTIC_PORT=9200 + es01: + image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1 + container_name: es01 + environment: + - node.name=es01 + - discovery.type=single-node + - indices.query.bool.max_clause_count=4096 + - bootstrap.memory_lock=true + - "ES_JAVA_OPTS=-Xms512m -Xmx512m" # no more than 32, remember to disable swap + #- "ES_JAVA_OPTS=-Xms8g -Xmx8g" # no more than 32, remember to disable swap + ulimits: + memlock: + soft: -1 + hard: -1 + volumes: + - es01:/usr/share/elasticsearch/data + ports: + - "9200:9200" + - "9300:9300" + #- 127.0.0.1:9200:9200 diff --git a/main.go b/main.go index 370d765..e4ed086 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "log" "net" "os" + "strings" "time" "github.com/akamensky/argparse" @@ -21,6 +22,8 @@ const ( defaultPort = "50051" defaultRPCUser = "rpcuser" defaultRPCPassword = "rpcpassword" + defaultEsHost = "http://localhost" + defaultEsPort = "9200" ) type loginCreds struct { @@ -39,6 +42,21 @@ func (c *loginCreds) RequireTransportSecurity() bool { } func parseArgs(searchRequest *pb.SearchRequest) *server.Args { + getenvironment := func(data []string, getkeyval func(item string) (key, val string)) map[string]string { + items := make(map[string]string) + for _, item := range data { + key, val := getkeyval(item) + items[key] = val + } + return items + } + environment := getenvironment(os.Environ(), func(item string) (key, val string) { + splits := strings.Split(item, "=") + key = splits[0] + val = splits[1] + return + }) + parser := argparse.NewParser("hub", "hub server and client") serveCmd := parser.NewCommand("serve", "start the hub server") @@ -63,7 +81,23 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args { log.Fatalln(parser.Usage(err)) } - args := &server.Args{Serve: false, Port: ":" + *port, User: *user, Pass: *pass} + + args := &server.Args{ + Serve: false, + Port: ":" + *port, + User: *user, + Pass: *pass, + EsHost: defaultEsHost, + EsPort: defaultEsPort, + } + + if esHost, ok := environment["ELASTIC_HOST"]; ok { + args.EsHost = esHost + } + + if esPort, ok := environment["ELASTIC_PORT"]; ok { + args.EsPort = esPort + } /* Verify no invalid argument combinations @@ -109,6 +143,9 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args { func main() { searchRequest := &pb.SearchRequest{} + // + //res := schema.ParseURL("@abc#1111") + //log.Println(res) args := parseArgs(searchRequest) diff --git a/schema/url.go b/schema/url.go index fcce04f..ae715e8 100644 --- a/schema/url.go +++ b/schema/url.go @@ -7,6 +7,7 @@ import ( "regexp" "strconv" "strings" + "unicode/utf16" ) type PathSegment struct { @@ -123,9 +124,16 @@ func ParseURL(url string) *URL { } func createUrlRegex() *regexp.Regexp { - //invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+" + d800 := []uint16{0xd800} + dfff := []uint16{0xdfff} + s1 := string(utf16.Decode(d800)) + s2 := string(utf16.Decode(dfff)) + log.Println(s1) + log.Println(s2) + //invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\\u0000-\\u0020\\uD800-\\uDFFF\\uFFFE-\\uFFFF]+" + invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020" + s1 + "-" + s2 + "\uFFFE-\uFFFF]+" //invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020-\uFFFE-\uFFFF]+" - invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "]+" + //invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "]+" named := func (name string, regex string) string { return "(?P<" + name + ">" + regex + ")" diff --git a/server/search.go b/server/search.go index 9b06246..1ee0024 100644 --- a/server/search.go +++ b/server/search.go @@ -341,7 +341,8 @@ func (s *Server) resolveUrl(ctx context.Context, rawUrl string) *urlResolution { func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) { var client *elastic.Client = nil if s.EsClient == nil { - tmpClient, err := elastic.NewClient(elastic.SetSniff(false)) + esUrl := s.Args.EsHost + ":" + s.Args.EsPort + tmpClient, err := elastic.NewClient(elastic.SetURL(esUrl), elastic.SetSniff(false)) if err != nil { return nil, err } @@ -351,8 +352,8 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, client = s.EsClient } - res := s.resolveUrl(ctx, "@abc#111") - log.Println(res) + //res := s.resolveUrl(ctx, "@abc#111") + //log.Println(res) claimTypes := map[string]int { "stream": 1, diff --git a/server/server.go b/server/server.go index 57af86c..e018794 100644 --- a/server/server.go +++ b/server/server.go @@ -24,6 +24,8 @@ type Args struct { Port string User string Pass string + EsHost string + EsPort string } type AccessDeniedErr struct {}