Add dockerfile and docker-compose with es

Dockerfile

@@ -0,0 +1,5 @@
+FROM debian:10-slim
+
+EXPOSE 50051
+COPY ./hub /hub
+ENTRYPOINT ["/hub", "serve"]

build.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+go build .
+docker build . -t lbry/hub:latest

docker-compose-hub-server.yml

@@ -0,0 +1,38 @@
+version: "3"
+
+volumes:
+  es01:
+
+services:
+  hub_server:
+    depends_on:
+      - es01
+    image: lbry/hub:latest
+    restart: always
+    ports:
+      - "50051:50051" # rpc port
+    environment:
+      #- TCP_PORT=50051 # should probably have these supported by the go server too
+      #- TCP_HOST=0.0.0.0
+      - ELASTIC_HOST=http://127.0.0.1
+      - ELASTIC_PORT=9200
+  es01:
+    image: docker.elastic.co/elasticsearch/elasticsearch:7.12.1
+    container_name: es01
+    environment:
+      - node.name=es01
+      - discovery.type=single-node
+      - indices.query.bool.max_clause_count=4096
+      - bootstrap.memory_lock=true
+      - "ES_JAVA_OPTS=-Xms512m -Xmx512m"  # no more than 32, remember to disable swap
+      #- "ES_JAVA_OPTS=-Xms8g -Xmx8g"  # no more than 32, remember to disable swap
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+    volumes:
+      - es01:/usr/share/elasticsearch/data
+    ports:
+      - "9200:9200"
+      - "9300:9300"
+      #- 127.0.0.1:9200:9200

main.go

@@ -7,6 +7,7 @@ import (
 	"log"
 	"net"
 	"os"
+	"strings"
 	"time"
 
 	"github.com/akamensky/argparse"
@@ -21,6 +22,8 @@ const (
 	defaultPort = "50051"
 	defaultRPCUser = "rpcuser"
 	defaultRPCPassword = "rpcpassword"
+	defaultEsHost = "http://localhost"
+	defaultEsPort = "9200"
 )
 
 type loginCreds struct {
@@ -39,6 +42,21 @@ func (c *loginCreds) RequireTransportSecurity() bool {
 }
 
 func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
+	getenvironment := func(data []string, getkeyval func(item string) (key, val string)) map[string]string {
+		items := make(map[string]string)
+		for _, item := range data {
+			key, val := getkeyval(item)
+			items[key] = val
+		}
+		return items
+	}
+	environment := getenvironment(os.Environ(), func(item string) (key, val string) {
+		splits := strings.Split(item, "=")
+		key = splits[0]
+		val = splits[1]
+		return
+	})
+
 	parser := argparse.NewParser("hub", "hub server and client")
 
 	serveCmd := parser.NewCommand("serve", "start the hub server")
@@ -63,7 +81,23 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
 		log.Fatalln(parser.Usage(err))
 	}
 
-	args := &server.Args{Serve: false, Port: ":" + *port, User: *user, Pass: *pass}
+
+	args := &server.Args{
+		Serve: false,
+		Port: ":" + *port,
+		User: *user,
+		Pass: *pass,
+		EsHost: defaultEsHost,
+		EsPort: defaultEsPort,
+	}
+
+	if esHost, ok := environment["ELASTIC_HOST"]; ok {
+		args.EsHost = esHost
+	}
+
+	if esPort, ok := environment["ELASTIC_PORT"]; ok {
+		args.EsPort = esPort
+	}
 
 	/*
 	Verify no invalid argument combinations
@@ -109,6 +143,9 @@ func parseArgs(searchRequest *pb.SearchRequest) *server.Args {
 
 func main() {
 	searchRequest := &pb.SearchRequest{}
+	//
+	//res := schema.ParseURL("@abc#1111")
+	//log.Println(res)
 
 	args := parseArgs(searchRequest)
 

schema/url.go

@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"strconv"
 	"strings"
+	"unicode/utf16"
 )
 
 type PathSegment struct {
@@ -123,9 +124,16 @@ func ParseURL(url string) *URL {
 }
 
 func createUrlRegex() *regexp.Regexp {
-	//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020\uD800-\uDFFF\uFFFE-\uFFFF]+"
+	d800 := []uint16{0xd800}
+	dfff := []uint16{0xdfff}
+	s1 := string(utf16.Decode(d800))
+	s2 := string(utf16.Decode(dfff))
+	log.Println(s1)
+	log.Println(s2)
+	//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\\u0000-\\u0020\\uD800-\\uDFFF\\uFFFE-\\uFFFF]+"
+	invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020" + s1 + "-" + s2 + "\uFFFE-\uFFFF]+"
 	//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "\u0000-\u0020-\uFFFE-\uFFFF]+"
-	invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "]+"
+	//invalidNamesRegex := "[^=&#:$@%?;\"/\\<>%{}|^~`\\[\\]" + "]+"
 
 	named := func (name string, regex string) string {
 		return "(?P<" + name + ">" + regex + ")"

server/search.go

@@ -341,7 +341,8 @@ func (s *Server) resolveUrl(ctx context.Context, rawUrl string) *urlResolution {
 func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs, error) {
 	var client *elastic.Client = nil
 	if s.EsClient == nil {
-		tmpClient, err := elastic.NewClient(elastic.SetSniff(false))
+		esUrl := s.Args.EsHost + ":" + s.Args.EsPort
+		tmpClient, err := elastic.NewClient(elastic.SetURL(esUrl), elastic.SetSniff(false))
 		if err != nil {
 			return nil, err
 		}
@@ -351,8 +352,8 @@ func (s *Server) Search(ctx context.Context, in *pb.SearchRequest) (*pb.Outputs,
 		client = s.EsClient
 	}
 
-	res := s.resolveUrl(ctx, "@abc#111")
-	log.Println(res)
+	//res := s.resolveUrl(ctx, "@abc#111")
+	//log.Println(res)
 
 	claimTypes := map[string]int {
 		"stream": 1,

server/server.go

@@ -24,6 +24,8 @@ type Args struct {
 	Port string
 	User string
 	Pass string
+	EsHost string
+	EsPort string
 }
 
 type AccessDeniedErr struct {}