From b7e67191298d48650470528dbf8ed3205575367a Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Sun, 8 Oct 2017 13:19:55 -0400 Subject: [PATCH 1/9] bittorrent: add initial request sanitizer --- bittorrent/bittorrent.go | 17 +++++---- bittorrent/request_sanitizer.go | 61 +++++++++++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 7 deletions(-) create mode 100644 bittorrent/request_sanitizer.go diff --git a/bittorrent/bittorrent.go b/bittorrent/bittorrent.go index 20c03e8..6bbead9 100644 --- a/bittorrent/bittorrent.go +++ b/bittorrent/bittorrent.go @@ -78,13 +78,16 @@ func (i InfoHash) String() string { // AnnounceRequest represents the parsed parameters from an announce request. type AnnounceRequest struct { - Event Event - InfoHash InfoHash - Compact bool - NumWant uint32 - Left uint64 - Downloaded uint64 - Uploaded uint64 + Event Event + InfoHash InfoHash + Compact bool + EventProvided bool + NumWantProvided bool + IPProvided bool + NumWant uint32 + Left uint64 + Downloaded uint64 + Uploaded uint64 Peer Params diff --git a/bittorrent/request_sanitizer.go b/bittorrent/request_sanitizer.go new file mode 100644 index 0000000..1e6c48a --- /dev/null +++ b/bittorrent/request_sanitizer.go @@ -0,0 +1,61 @@ +package bittorrent + +import ( + "net" + + log "github.com/sirupsen/logrus" +) + +// ErrInvalidIP indicates an invalid IP for an Announce. +var ErrInvalidIP = ClientError("invalid IP") + +// RequestSanitizer is used to replace unreasonable values in requests parsed +// from a frontend into sane values. +type RequestSanitizer struct { + MaxNumWant uint32 `yaml:"max_numwant"` + DefaultNumWant uint32 `yaml:"default_numwant"` + MaxScrapeInfoHashes uint32 `yaml:"max_scrape_infohashes"` +} + +// SanitizeAnnounce enforces a max and default NumWant and coerces the peer's +// IP address into the proper format. +func (rs *RequestSanitizer) SanitizeAnnounce(r *AnnounceRequest) error { + if !r.NumWantProvided { + r.NumWant = rs.DefaultNumWant + } else if r.NumWant > rs.MaxNumWant { + r.NumWant = rs.MaxNumWant + } + + if ip := r.Peer.IP.To4(); ip != nil { + r.Peer.IP.IP = ip + r.Peer.IP.AddressFamily = IPv4 + } else if len(r.Peer.IP.IP) == net.IPv6len { // implies r.Peer.IP.To4() == nil + r.Peer.IP.AddressFamily = IPv6 + } else { + return ErrInvalidIP + } + + log.Debug("sanitized announce", rs, r) + return nil +} + +// SanitizeScrape enforces a max number of infohashes for a single scrape +// request. +func (rs *RequestSanitizer) SanitizeScrape(r *ScrapeRequest) error { + if len(r.InfoHashes) > int(rs.MaxScrapeInfoHashes) { + r.InfoHashes = r.InfoHashes[:rs.MaxScrapeInfoHashes] + } + + log.Debug("sanitized scrape", rs, r) + return nil +} + +// LogFields renders the request sanitizer's configuration as a set of loggable +// fields. +func (rs *RequestSanitizer) LogFields() log.Fields { + return log.Fields{ + "maxNumWant": rs.MaxNumWant, + "defaultNumWant": rs.DefaultNumWant, + "maxScrapeInfohashes": rs.MaxScrapeInfoHashes, + } +} From 6dee48ce174393b8934dfa02f7378643dc8708a8 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Sun, 8 Oct 2017 17:16:02 -0400 Subject: [PATCH 2/9] frontend/http: add request sanitization --- frontend/http/frontend.go | 14 ++++--- frontend/http/parser.go | 77 +++++++++++++++++++++++++-------------- 2 files changed, 58 insertions(+), 33 deletions(-) diff --git a/frontend/http/frontend.go b/frontend/http/frontend.go index c347d10..dc10dab 100644 --- a/frontend/http/frontend.go +++ b/frontend/http/frontend.go @@ -65,11 +65,10 @@ type Config struct { Addr string `yaml:"addr"` ReadTimeout time.Duration `yaml:"read_timeout"` WriteTimeout time.Duration `yaml:"write_timeout"` - AllowIPSpoofing bool `yaml:"allow_ip_spoofing"` - RealIPHeader string `yaml:"real_ip_header"` TLSCertPath string `yaml:"tls_cert_path"` TLSKeyPath string `yaml:"tls_key_path"` EnableRequestTiming bool `yaml:"enable_request_timing"` + ParseOptions `yaml:",inline"` } // LogFields renders the current config as a set of Logrus fields. @@ -78,11 +77,14 @@ func (cfg Config) LogFields() log.Fields { "addr": cfg.Addr, "readTimeout": cfg.ReadTimeout, "writeTimeout": cfg.WriteTimeout, - "allowIPSpoofing": cfg.AllowIPSpoofing, - "realIPHeader": cfg.RealIPHeader, "tlsCertPath": cfg.TLSCertPath, "tlsKeyPath": cfg.TLSKeyPath, "enableRequestTiming": cfg.EnableRequestTiming, + "allowIPSpoofing": cfg.AllowIPSpoofing, + "realIPHeader": cfg.RealIPHeader, + "maxNumWant": cfg.MaxNumWant, + "defaultNumWant": cfg.DefaultNumWant, + "maxScrapeInfohashes": cfg.MaxScrapeInfoHashes, } } @@ -219,7 +221,7 @@ func (f *Frontend) announceRoute(w http.ResponseWriter, r *http.Request, _ httpr } }() - req, err := ParseAnnounce(r, f.RealIPHeader, f.AllowIPSpoofing) + req, err := ParseAnnounce(r, f.ParseOptions) if err != nil { WriteError(w, err) return @@ -258,7 +260,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, _ httprou } }() - req, err := ParseScrape(r) + req, err := ParseScrape(r, f.ParseOptions) if err != nil { WriteError(w, err) return diff --git a/frontend/http/parser.go b/frontend/http/parser.go index 7f4e2ca..29178bd 100644 --- a/frontend/http/parser.go +++ b/frontend/http/parser.go @@ -7,12 +7,19 @@ import ( "github.com/chihaya/chihaya/bittorrent" ) -// ParseAnnounce parses an bittorrent.AnnounceRequest from an http.Request. +// ParseOptions is the configuration used to parse an Announce Request. // -// If allowIPSpoofing is true, IPs provided via params will be used. -// If realIPHeader is not empty string, the first value of the HTTP Header with +// If AllowIPSpoofing is true, IPs provided via BitTorrent params will be used. +// If RealIPHeader is not empty string, the value of the first HTTP Header with // that name will be used. -func ParseAnnounce(r *http.Request, realIPHeader string, allowIPSpoofing bool) (*bittorrent.AnnounceRequest, error) { +type ParseOptions struct { + AllowIPSpoofing bool `yaml:"allowIPSpoofing"` + RealIPHeader string `yaml:"realIPHeader"` + bittorrent.RequestSanitizer `yaml:",inline"` +} + +// ParseAnnounce parses an bittorrent.AnnounceRequest from an http.Request. +func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequest, error) { qp, err := bittorrent.ParseURLData(r.RequestURI) if err != nil { return nil, err @@ -20,15 +27,23 @@ func ParseAnnounce(r *http.Request, realIPHeader string, allowIPSpoofing bool) ( request := &bittorrent.AnnounceRequest{Params: qp} - eventStr, _ := qp.String("event") - request.Event, err = bittorrent.NewEvent(eventStr) - if err != nil { - return nil, bittorrent.ClientError("failed to provide valid client event") + // Attempt to parse the event from the request. + var eventStr string + eventStr, request.EventProvided = qp.String("event") + if request.EventProvided { + request.Event, err = bittorrent.NewEvent(eventStr) + if err != nil { + return nil, bittorrent.ClientError("failed to provide valid client event") + } + } else { + request.Event = bittorrent.None } + // Determine if the client expects a compact response. compactStr, _ := qp.String("compact") request.Compact = compactStr != "" && compactStr != "0" + // Parse the infohash from the request. infoHashes := qp.InfoHashes() if len(infoHashes) < 1 { return nil, bittorrent.ClientError("no info_hash parameter supplied") @@ -38,6 +53,7 @@ func ParseAnnounce(r *http.Request, realIPHeader string, allowIPSpoofing bool) ( } request.InfoHash = infoHashes[0] + // Parse the PeerID from the request. peerID, ok := qp.String("peer_id") if !ok { return nil, bittorrent.ClientError("failed to parse parameter: peer_id") @@ -47,43 +63,54 @@ func ParseAnnounce(r *http.Request, realIPHeader string, allowIPSpoofing bool) ( } request.Peer.ID = bittorrent.PeerIDFromString(peerID) + // Determine the number of remaining pieces for the client. request.Left, err = qp.Uint64("left") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: left") } + // Determine the number of pieces downloaded by the client. request.Downloaded, err = qp.Uint64("downloaded") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: downloaded") } + // Determine the number of pieces shared by the client. request.Uploaded, err = qp.Uint64("uploaded") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: uploaded") } + // Determine the number of peers the client wants in the response. numwant, err := qp.Uint64("numwant") if err != nil && err != bittorrent.ErrKeyNotFound { return nil, bittorrent.ClientError("failed to parse parameter: numwant") } request.NumWant = uint32(numwant) + request.NumWantProvided = err == nil + // Parse the port where the client is listening. port, err := qp.Uint64("port") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: port") } request.Peer.Port = uint16(port) - request.Peer.IP.IP = requestedIP(r, qp, realIPHeader, allowIPSpoofing) + // Parse the IP address where the client is listening. + request.Peer.IP.IP, request.IPProvided = requestedIP(r, qp, opts) if request.Peer.IP.IP == nil { return nil, bittorrent.ClientError("failed to parse peer IP address") } + if err := opts.SanitizeAnnounce(request); err != nil { + return nil, err + } + return request, nil } // ParseScrape parses an bittorrent.ScrapeRequest from an http.Request. -func ParseScrape(r *http.Request) (*bittorrent.ScrapeRequest, error) { +func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest, error) { qp, err := bittorrent.ParseURLData(r.RequestURI) if err != nil { return nil, err @@ -99,39 +126,35 @@ func ParseScrape(r *http.Request) (*bittorrent.ScrapeRequest, error) { Params: qp, } + if err := opts.SanitizeScrape(request); err != nil { + return nil, err + } + return request, nil } // requestedIP determines the IP address for a BitTorrent client request. -// -// If allowIPSpoofing is true, IPs provided via params will be used. -// If realIPHeader is not empty string, the first value of the HTTP Header with -// that name will be used. -func requestedIP(r *http.Request, p bittorrent.Params, realIPHeader string, allowIPSpoofing bool) net.IP { - if allowIPSpoofing { +func requestedIP(r *http.Request, p bittorrent.Params, opts ParseOptions) (ip net.IP, provided bool) { + if opts.AllowIPSpoofing { if ipstr, ok := p.String("ip"); ok { - ip := net.ParseIP(ipstr) - return ip + return net.ParseIP(ipstr), true } if ipstr, ok := p.String("ipv4"); ok { - ip := net.ParseIP(ipstr) - return ip + return net.ParseIP(ipstr), true } if ipstr, ok := p.String("ipv6"); ok { - ip := net.ParseIP(ipstr) - return ip + return net.ParseIP(ipstr), true } } - if realIPHeader != "" { - if ips, ok := r.Header[realIPHeader]; ok && len(ips) > 0 { - ip := net.ParseIP(ips[0]) - return ip + if opts.RealIPHeader != "" { + if ips, ok := r.Header[opts.RealIPHeader]; ok && len(ips) > 0 { + return net.ParseIP(ips[0]), false } } host, _, _ := net.SplitHostPort(r.RemoteAddr) - return net.ParseIP(host) + return net.ParseIP(host), false } From 47b5e67345f6e271ce214e66d1843f411ed28127 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Sun, 8 Oct 2017 17:35:50 -0400 Subject: [PATCH 3/9] frontend/udp: add request sanitization --- frontend/udp/frontend.go | 11 +++++--- frontend/udp/parser.go | 58 +++++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 23 deletions(-) diff --git a/frontend/udp/frontend.go b/frontend/udp/frontend.go index 4ecbf85..e50d182 100644 --- a/frontend/udp/frontend.go +++ b/frontend/udp/frontend.go @@ -70,8 +70,8 @@ type Config struct { Addr string `yaml:"addr"` PrivateKey string `yaml:"private_key"` MaxClockSkew time.Duration `yaml:"max_clock_skew"` - AllowIPSpoofing bool `yaml:"allow_ip_spoofing"` EnableRequestTiming bool `yaml:"enable_request_timing"` + ParseOptions `yaml:",inline"` } // LogFields renders the current config as a set of Logrus fields. @@ -80,8 +80,11 @@ func (cfg Config) LogFields() log.Fields { "addr": cfg.Addr, "privateKey": cfg.PrivateKey, "maxClockSkew": cfg.MaxClockSkew, - "allowIPSpoofing": cfg.AllowIPSpoofing, "enableRequestTiming": cfg.EnableRequestTiming, + "allowIPSpoofing": cfg.AllowIPSpoofing, + "maxNumWant": cfg.MaxNumWant, + "defaultNumWant": cfg.DefaultNumWant, + "maxScrapeInfohashes": cfg.MaxScrapeInfoHashes, } } @@ -278,7 +281,7 @@ func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string actionName = "announce" var req *bittorrent.AnnounceRequest - req, err = ParseAnnounce(r, t.AllowIPSpoofing, actionID == announceV6ActionID) + req, err = ParseAnnounce(r, actionID == announceV6ActionID, t.ParseOptions) if err != nil { WriteError(w, txID, err) return @@ -302,7 +305,7 @@ func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string actionName = "scrape" var req *bittorrent.ScrapeRequest - req, err = ParseScrape(r) + req, err = ParseScrape(r, t.ParseOptions) if err != nil { WriteError(w, txID, err) return diff --git a/frontend/udp/parser.go b/frontend/udp/parser.go index 9bc453c..e35e3b4 100644 --- a/frontend/udp/parser.go +++ b/frontend/udp/parser.go @@ -45,14 +45,19 @@ var ( errUnknownOptionType = bittorrent.ClientError("unknown option type") ) +// ParseOptions is the configuration used to parse an Announce Request. +// +// If AllowIPSpoofing is true, IPs provided via params will be used. +type ParseOptions struct { + AllowIPSpoofing bool `yaml:"allowIPSpoofing"` + bittorrent.RequestSanitizer `yaml:",inline"` +} + // ParseAnnounce parses an AnnounceRequest from a UDP request. // -// If allowIPSpoofing is true, IPs provided via params will be used. -// -// If v6 is true the announce will be parsed as an IPv6 announce "the -// opentracker way", see +// If v6 is true, the announce is parsed the "opentracker way": // http://opentracker.blog.h3q.com/2007/12/28/the-ipv6-situation/ -func ParseAnnounce(r Request, allowIPSpoofing, v6 bool) (*bittorrent.AnnounceRequest, error) { +func ParseAnnounce(r Request, v6 bool, opts ParseOptions) (*bittorrent.AnnounceRequest, error) { ipEnd := 84 + net.IPv4len if v6 { ipEnd = 84 + net.IPv6len @@ -74,12 +79,14 @@ func ParseAnnounce(r Request, allowIPSpoofing, v6 bool) (*bittorrent.AnnounceReq } ip := r.IP + ipProvided := false ipbytes := r.Packet[84:ipEnd] - if allowIPSpoofing { + if opts.AllowIPSpoofing { // Make sure the bytes are copied to a new slice. copy(ip, net.IP(ipbytes)) + ipProvided = true } - if !allowIPSpoofing && r.IP == nil { + if !opts.AllowIPSpoofing && r.IP == nil { // We have no IP address to fallback on. return nil, errMalformedIP } @@ -92,20 +99,29 @@ func ParseAnnounce(r Request, allowIPSpoofing, v6 bool) (*bittorrent.AnnounceReq return nil, err } - return &bittorrent.AnnounceRequest{ - Event: eventIDs[eventID], - InfoHash: bittorrent.InfoHashFromBytes(infohash), - NumWant: uint32(numWant), - Left: left, - Downloaded: downloaded, - Uploaded: uploaded, + request := &bittorrent.AnnounceRequest{ + Event: eventIDs[eventID], + InfoHash: bittorrent.InfoHashFromBytes(infohash), + NumWant: uint32(numWant), + Left: left, + Downloaded: downloaded, + Uploaded: uploaded, + IPProvided: ipProvided, + NumWantProvided: true, + EventProvided: true, Peer: bittorrent.Peer{ ID: bittorrent.PeerIDFromBytes(peerID), IP: bittorrent.IP{IP: ip}, Port: port, }, Params: params, - }, nil + } + + if err := opts.SanitizeAnnounce(request); err != nil { + return nil, err + } + + return request, nil } type buffer struct { @@ -170,7 +186,7 @@ func handleOptionalParameters(packet []byte) (bittorrent.Params, error) { } // ParseScrape parses a ScrapeRequest from a UDP request. -func ParseScrape(r Request) (*bittorrent.ScrapeRequest, error) { +func ParseScrape(r Request, opts ParseOptions) (*bittorrent.ScrapeRequest, error) { // If a scrape isn't at least 36 bytes long, it's malformed. if len(r.Packet) < 36 { return nil, errMalformedPacket @@ -190,7 +206,11 @@ func ParseScrape(r Request) (*bittorrent.ScrapeRequest, error) { r.Packet = r.Packet[20:] } - return &bittorrent.ScrapeRequest{ - InfoHashes: infohashes, - }, nil + // Sanitize the request. + request := &bittorrent.ScrapeRequest{InfoHashes: infohashes} + if err := opts.SanitizeScrape(request); err != nil { + return nil, err + } + + return request, nil } From 134744a4841e5cac54fc4d301cf34707385b1180 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Sun, 8 Oct 2017 17:41:00 -0400 Subject: [PATCH 4/9] middleware: remove sanitization mw --- middleware/hooks.go | 52 ----------------------------------- middleware/middleware.go | 14 ++-------- middleware/middleware_test.go | 11 -------- 3 files changed, 3 insertions(+), 74 deletions(-) diff --git a/middleware/hooks.go b/middleware/hooks.go index 3336917..a77db86 100644 --- a/middleware/hooks.go +++ b/middleware/hooks.go @@ -2,8 +2,6 @@ package middleware import ( "context" - "errors" - "net" "github.com/chihaya/chihaya/bittorrent" "github.com/chihaya/chihaya/storage" @@ -67,56 +65,6 @@ func (h *swarmInteractionHook) HandleScrape(ctx context.Context, _ *bittorrent.S return ctx, nil } -// ErrInvalidIP indicates an invalid IP for an Announce. -var ErrInvalidIP = errors.New("invalid IP") - -// sanitizationHook enforces semantic assumptions about requests that may have -// not been accounted for in a tracker frontend. -// -// The SanitizationHook performs the following checks: -// - maxNumWant: Checks whether the numWant parameter of an announce is below -// a limit. Sets it to the limit if the value is higher. -// - defaultNumWant: Checks whether the numWant parameter of an announce is -// zero. Sets it to the default if it is. -// - IP sanitization: Checks whether the announcing Peer's IP address is either -// IPv4 or IPv6. Returns ErrInvalidIP if the address is neither IPv4 nor -// IPv6. Sets the Peer.AddressFamily field accordingly. Truncates IPv4 -// addresses to have a length of 4 bytes. -type sanitizationHook struct { - maxNumWant uint32 - defaultNumWant uint32 - maxScrapeInfoHashes uint32 -} - -func (h *sanitizationHook) HandleAnnounce(ctx context.Context, req *bittorrent.AnnounceRequest, resp *bittorrent.AnnounceResponse) (context.Context, error) { - if req.NumWant > h.maxNumWant { - req.NumWant = h.maxNumWant - } - - if req.NumWant == 0 { - req.NumWant = h.defaultNumWant - } - - if ip := req.Peer.IP.To4(); ip != nil { - req.Peer.IP.IP = ip - req.Peer.IP.AddressFamily = bittorrent.IPv4 - } else if len(req.Peer.IP.IP) == net.IPv6len { // implies req.Peer.IP.To4() == nil - req.Peer.IP.AddressFamily = bittorrent.IPv6 - } else { - return ctx, ErrInvalidIP - } - - return ctx, nil -} - -func (h *sanitizationHook) HandleScrape(ctx context.Context, req *bittorrent.ScrapeRequest, resp *bittorrent.ScrapeResponse) (context.Context, error) { - if len(req.InfoHashes) > int(h.maxScrapeInfoHashes) { - req.InfoHashes = req.InfoHashes[:h.maxScrapeInfoHashes] - } - - return ctx, nil -} - type skipResponseHook struct{} // SkipResponseHookKey is a key for the context of an Announce or Scrape to diff --git a/middleware/middleware.go b/middleware/middleware.go index 14c2845..0fdabb8 100644 --- a/middleware/middleware.go +++ b/middleware/middleware.go @@ -15,10 +15,7 @@ import ( // Config holds the configuration common across all middleware. type Config struct { - AnnounceInterval time.Duration `yaml:"announce_interval"` - MaxNumWant uint32 `yaml:"max_numwant"` - DefaultNumWant uint32 `yaml:"default_numwant"` - MaxScrapeInfoHashes uint32 `yaml:"max_scrape_infohashes"` + AnnounceInterval time.Duration `yaml:"announce_interval"` } var _ frontend.TrackerLogic = &Logic{} @@ -26,17 +23,12 @@ var _ frontend.TrackerLogic = &Logic{} // NewLogic creates a new instance of a TrackerLogic that executes the provided // middleware hooks. func NewLogic(cfg Config, peerStore storage.PeerStore, preHooks, postHooks []Hook) *Logic { - l := &Logic{ + return &Logic{ announceInterval: cfg.AnnounceInterval, peerStore: peerStore, - preHooks: []Hook{&sanitizationHook{cfg.MaxNumWant, cfg.DefaultNumWant, cfg.MaxScrapeInfoHashes}}, + preHooks: append(preHooks, &responseHook{store: peerStore}), postHooks: append(postHooks, &swarmInteractionHook{store: peerStore}), } - - l.preHooks = append(l.preHooks, preHooks...) - l.preHooks = append(l.preHooks, &responseHook{store: peerStore}) - - return l } // Logic is an implementation of the TrackerLogic that functions by diff --git a/middleware/middleware_test.go b/middleware/middleware_test.go index 7821d61..9976815 100644 --- a/middleware/middleware_test.go +++ b/middleware/middleware_test.go @@ -80,15 +80,4 @@ func BenchmarkHookOverhead(b *testing.B) { benchHookListV6(b, nopHooks) }) } - - var sanHooks hookList - for i := 1; i < 4; i++ { - sanHooks = append(sanHooks, &sanitizationHook{maxNumWant: 50}) - b.Run(fmt.Sprintf("%dsanitation-v4", i), func(b *testing.B) { - benchHookListV4(b, sanHooks) - }) - b.Run(fmt.Sprintf("%dsanitation-v6", i), func(b *testing.B) { - benchHookListV6(b, sanHooks) - }) - } } From ce43a09956007f673d516a672457819eff6eedbb Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Sun, 8 Oct 2017 17:44:52 -0400 Subject: [PATCH 5/9] *: add sanitization example config --- example_config.yaml | 43 +++++++++++++++++++++++++++++++------------ 1 file changed, 31 insertions(+), 12 deletions(-) diff --git a/example_config.yaml b/example_config.yaml index 8b6bffa..4116a99 100644 --- a/example_config.yaml +++ b/example_config.yaml @@ -24,14 +24,6 @@ chihaya: # BitTorrent traffic. addr: "0.0.0.0:6881" - # When enabled, the IP address used to connect to the tracker will not - # override the value clients advertise as their IP address. - allow_ip_spoofing: false - - # The HTTP Header containing the IP address of the client. - # This is only necessary if using a reverse proxy. - real_ip_header: "x-real-ip" - # The path to the required files to listen via HTTPS. tls_cert_path: "" tls_key_path: "" @@ -44,6 +36,23 @@ chihaya: # Disabling this should increase performance/decrease load. enable_request_timing: false + # When enabled, the IP address used to connect to the tracker will not + # override the value clients advertise as their IP address. + allow_ip_spoofing: false + + # The HTTP Header containing the IP address of the client. + # This is only necessary if using a reverse proxy. + real_ip_header: "x-real-ip" + + # The maximum number of peers returned for an individual request. + max_numwant: 100 + + # The default number of peers returned for an individual request. + default_numwant: 50 + + # The maximum number of infohashes that can be scraped in one request. + max_scrape_infohashes: 50 + # This block defines configuration for the tracker's UDP interface. # If you do not wish to run this, delete this section. udp: @@ -51,10 +60,6 @@ chihaya: # BitTorrent traffic. addr: "0.0.0.0:6881" - # When enabled, the IP address used to connect to the tracker will not - # override the value clients advertise as their IP address. - allow_ip_spoofing: false - # The leeway for a timestamp on a connection ID. max_clock_skew: 10s @@ -65,6 +70,20 @@ chihaya: # Disabling this should increase performance/decrease load. enable_request_timing: false + # When enabled, the IP address used to connect to the tracker will not + # override the value clients advertise as their IP address. + allow_ip_spoofing: false + + # The maximum number of peers returned for an individual request. + max_numwant: 100 + + # The default number of peers returned for an individual request. + default_numwant: 50 + + # The maximum number of infohashes that can be scraped in one request. + max_scrape_infohashes: 50 + + # This block defines configuration used for the storage of peer data. storage: name: memory From 66e12c6684a9770fab1c0a3c96b46ed4d37e69dc Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Oct 2017 22:02:06 -0400 Subject: [PATCH 6/9] bittorrent: add String() and LogFields() --- bittorrent/bittorrent.go | 58 ++++++++++++++++++++++++++++++++++------ 1 file changed, 50 insertions(+), 8 deletions(-) diff --git a/bittorrent/bittorrent.go b/bittorrent/bittorrent.go index 6bbead9..cd0bd2a 100644 --- a/bittorrent/bittorrent.go +++ b/bittorrent/bittorrent.go @@ -93,6 +93,24 @@ type AnnounceRequest struct { Params } +// LogFields renders the current response as a set of log fields. +func (r AnnounceRequest) LogFields() log.Fields { + return log.Fields{ + "event": r.Event, + "infoHash": r.InfoHash, + "compact": r.Compact, + "eventProvided": r.EventProvided, + "numWantProvided": r.NumWantProvided, + "ipProvided": r.IPProvided, + "numWant": r.NumWant, + "left": r.Left, + "downloaded": r.Downloaded, + "uploaded": r.Uploaded, + "peer": r.Peer, + "params": r.Params, + } +} + // AnnounceResponse represents the parameters used to create an announce // response. type AnnounceResponse struct { @@ -105,15 +123,15 @@ type AnnounceResponse struct { IPv6Peers []Peer } -// LogFields renders the current response as a set of Logrus fields. -func (ar AnnounceResponse) LogFields() log.Fields { +// LogFields renders the current response as a set of log fields. +func (r AnnounceResponse) LogFields() log.Fields { return log.Fields{ - "compact": ar.Compact, - "complete": ar.Complete, - "interval": ar.Interval, - "minInterval": ar.MinInterval, - "ipv4Peers": ar.IPv4Peers, - "ipv6Peers": ar.IPv6Peers, + "compact": r.Compact, + "complete": r.Complete, + "interval": r.Interval, + "minInterval": r.MinInterval, + "ipv4Peers": r.IPv4Peers, + "ipv6Peers": r.IPv6Peers, } } @@ -124,6 +142,15 @@ type ScrapeRequest struct { Params Params } +// LogFields renders the current response as a set of log fields. +func (r ScrapeRequest) LogFields() log.Fields { + return log.Fields{ + "addressFamily": r.AddressFamily, + "infoHashes": r.InfoHashes, + "params": r.Params, + } +} + // ScrapeResponse represents the parameters used to create a scrape response. // // The Scrapes must be in the same order as the InfoHashes in the corresponding @@ -150,6 +177,17 @@ type Scrape struct { // AddressFamily is the address family of an IP address. type AddressFamily uint8 +func (af AddressFamily) String() string { + switch af { + case IPv4: + return "IPv4" + case IPv6: + return "IPv6" + default: + panic("tried to print unknown AddressFamily") + } +} + // AddressFamily constants. const ( IPv4 AddressFamily = iota @@ -162,6 +200,10 @@ type IP struct { AddressFamily } +func (ip IP) String() string { + return ip.IP.String() +} + // Peer represents the connection details of a peer that is returned in an // announce response. type Peer struct { From ca823e0e5f8511717a2d7141f7455a7551b3cb8c Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Oct 2017 22:02:45 -0400 Subject: [PATCH 7/9] frontend: update to use non-object sanization --- bittorrent/request_sanitizer.go | 61 --------------------------------- bittorrent/sanitize.go | 48 ++++++++++++++++++++++++++ frontend/http/frontend.go | 7 ++-- frontend/http/parser.go | 12 ++++--- frontend/udp/frontend.go | 7 ++-- frontend/udp/parser.go | 10 +++--- 6 files changed, 65 insertions(+), 80 deletions(-) delete mode 100644 bittorrent/request_sanitizer.go create mode 100644 bittorrent/sanitize.go diff --git a/bittorrent/request_sanitizer.go b/bittorrent/request_sanitizer.go deleted file mode 100644 index 1e6c48a..0000000 --- a/bittorrent/request_sanitizer.go +++ /dev/null @@ -1,61 +0,0 @@ -package bittorrent - -import ( - "net" - - log "github.com/sirupsen/logrus" -) - -// ErrInvalidIP indicates an invalid IP for an Announce. -var ErrInvalidIP = ClientError("invalid IP") - -// RequestSanitizer is used to replace unreasonable values in requests parsed -// from a frontend into sane values. -type RequestSanitizer struct { - MaxNumWant uint32 `yaml:"max_numwant"` - DefaultNumWant uint32 `yaml:"default_numwant"` - MaxScrapeInfoHashes uint32 `yaml:"max_scrape_infohashes"` -} - -// SanitizeAnnounce enforces a max and default NumWant and coerces the peer's -// IP address into the proper format. -func (rs *RequestSanitizer) SanitizeAnnounce(r *AnnounceRequest) error { - if !r.NumWantProvided { - r.NumWant = rs.DefaultNumWant - } else if r.NumWant > rs.MaxNumWant { - r.NumWant = rs.MaxNumWant - } - - if ip := r.Peer.IP.To4(); ip != nil { - r.Peer.IP.IP = ip - r.Peer.IP.AddressFamily = IPv4 - } else if len(r.Peer.IP.IP) == net.IPv6len { // implies r.Peer.IP.To4() == nil - r.Peer.IP.AddressFamily = IPv6 - } else { - return ErrInvalidIP - } - - log.Debug("sanitized announce", rs, r) - return nil -} - -// SanitizeScrape enforces a max number of infohashes for a single scrape -// request. -func (rs *RequestSanitizer) SanitizeScrape(r *ScrapeRequest) error { - if len(r.InfoHashes) > int(rs.MaxScrapeInfoHashes) { - r.InfoHashes = r.InfoHashes[:rs.MaxScrapeInfoHashes] - } - - log.Debug("sanitized scrape", rs, r) - return nil -} - -// LogFields renders the request sanitizer's configuration as a set of loggable -// fields. -func (rs *RequestSanitizer) LogFields() log.Fields { - return log.Fields{ - "maxNumWant": rs.MaxNumWant, - "defaultNumWant": rs.DefaultNumWant, - "maxScrapeInfohashes": rs.MaxScrapeInfoHashes, - } -} diff --git a/bittorrent/sanitize.go b/bittorrent/sanitize.go new file mode 100644 index 0000000..9cc7402 --- /dev/null +++ b/bittorrent/sanitize.go @@ -0,0 +1,48 @@ +package bittorrent + +import ( + "net" + + "github.com/chihaya/chihaya/pkg/log" +) + +// ErrInvalidIP indicates an invalid IP for an Announce. +var ErrInvalidIP = ClientError("invalid IP") + +// SanitizeAnnounce enforces a max and default NumWant and coerces the peer's +// IP address into the proper format. +func SanitizeAnnounce(r *AnnounceRequest, maxNumWant, defaultNumWant uint32) error { + if !r.NumWantProvided { + r.NumWant = defaultNumWant + } else if r.NumWant > maxNumWant { + r.NumWant = maxNumWant + } + + if ip := r.Peer.IP.To4(); ip != nil { + r.Peer.IP.IP = ip + r.Peer.IP.AddressFamily = IPv4 + } else if len(r.Peer.IP.IP) == net.IPv6len { // implies r.Peer.IP.To4() == nil + r.Peer.IP.AddressFamily = IPv6 + } else { + return ErrInvalidIP + } + + log.Debug("sanitized announce", r, log.Fields{ + "maxNumWant": maxNumWant, + "defaultNumWant": defaultNumWant, + }) + return nil +} + +// SanitizeScrape enforces a max number of infohashes for a single scrape +// request. +func SanitizeScrape(r *ScrapeRequest, maxScrapeInfoHashes uint32) error { + if len(r.InfoHashes) > int(maxScrapeInfoHashes) { + r.InfoHashes = r.InfoHashes[:maxScrapeInfoHashes] + } + + log.Debug("sanitized scrape", r, log.Fields{ + "maxScrapeInfoHashes": maxScrapeInfoHashes, + }) + return nil +} diff --git a/frontend/http/frontend.go b/frontend/http/frontend.go index dc10dab..1ed1323 100644 --- a/frontend/http/frontend.go +++ b/frontend/http/frontend.go @@ -21,9 +21,6 @@ func init() { prometheus.MustRegister(promResponseDurationMilliseconds) } -// ErrInvalidIP indicates an invalid IP. -var ErrInvalidIP = bittorrent.ClientError("invalid IP") - var promResponseDurationMilliseconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "chihaya_http_response_duration_milliseconds", @@ -84,7 +81,7 @@ func (cfg Config) LogFields() log.Fields { "realIPHeader": cfg.RealIPHeader, "maxNumWant": cfg.MaxNumWant, "defaultNumWant": cfg.DefaultNumWant, - "maxScrapeInfohashes": cfg.MaxScrapeInfoHashes, + "maxScrapeInfoHashes": cfg.MaxScrapeInfoHashes, } } @@ -280,7 +277,7 @@ func (f *Frontend) scrapeRoute(w http.ResponseWriter, r *http.Request, _ httprou req.AddressFamily = bittorrent.IPv6 } else { log.Error("http: invalid IP: neither v4 nor v6", log.Fields{"RemoteAddr": r.RemoteAddr}) - WriteError(w, ErrInvalidIP) + WriteError(w, bittorrent.ErrInvalidIP) return } af = new(bittorrent.AddressFamily) diff --git a/frontend/http/parser.go b/frontend/http/parser.go index 29178bd..c463936 100644 --- a/frontend/http/parser.go +++ b/frontend/http/parser.go @@ -13,9 +13,11 @@ import ( // If RealIPHeader is not empty string, the value of the first HTTP Header with // that name will be used. type ParseOptions struct { - AllowIPSpoofing bool `yaml:"allowIPSpoofing"` - RealIPHeader string `yaml:"realIPHeader"` - bittorrent.RequestSanitizer `yaml:",inline"` + AllowIPSpoofing bool `yaml:"allow_ip_spoofing"` + RealIPHeader string `yaml:"real_ip_header"` + MaxNumWant uint32 `yaml:"max_numwant"` + DefaultNumWant uint32 `yaml:"default_numwant"` + MaxScrapeInfoHashes uint32 `yaml:"max_scrape_infohashes"` } // ParseAnnounce parses an bittorrent.AnnounceRequest from an http.Request. @@ -102,7 +104,7 @@ func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequ return nil, bittorrent.ClientError("failed to parse peer IP address") } - if err := opts.SanitizeAnnounce(request); err != nil { + if err := bittorrent.SanitizeAnnounce(request, opts.MaxNumWant, opts.DefaultNumWant); err != nil { return nil, err } @@ -126,7 +128,7 @@ func ParseScrape(r *http.Request, opts ParseOptions) (*bittorrent.ScrapeRequest, Params: qp, } - if err := opts.SanitizeScrape(request); err != nil { + if err := bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes); err != nil { return nil, err } diff --git a/frontend/udp/frontend.go b/frontend/udp/frontend.go index e50d182..54a3b23 100644 --- a/frontend/udp/frontend.go +++ b/frontend/udp/frontend.go @@ -26,9 +26,6 @@ func init() { prometheus.MustRegister(promResponseDurationMilliseconds) } -// ErrInvalidIP indicates an invalid IP. -var ErrInvalidIP = bittorrent.ClientError("invalid IP") - var promResponseDurationMilliseconds = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "chihaya_udp_response_duration_milliseconds", @@ -84,7 +81,7 @@ func (cfg Config) LogFields() log.Fields { "allowIPSpoofing": cfg.AllowIPSpoofing, "maxNumWant": cfg.MaxNumWant, "defaultNumWant": cfg.DefaultNumWant, - "maxScrapeInfohashes": cfg.MaxScrapeInfoHashes, + "maxScrapeInfoHashes": cfg.MaxScrapeInfoHashes, } } @@ -317,7 +314,7 @@ func (t *Frontend) handleRequest(r Request, w ResponseWriter) (actionName string req.AddressFamily = bittorrent.IPv6 } else { log.Error("udp: invalid IP: neither v4 nor v6", log.Fields{"IP": r.IP}) - WriteError(w, txID, ErrInvalidIP) + WriteError(w, txID, bittorrent.ErrInvalidIP) return } af = new(bittorrent.AddressFamily) diff --git a/frontend/udp/parser.go b/frontend/udp/parser.go index e35e3b4..5a668f6 100644 --- a/frontend/udp/parser.go +++ b/frontend/udp/parser.go @@ -49,8 +49,10 @@ var ( // // If AllowIPSpoofing is true, IPs provided via params will be used. type ParseOptions struct { - AllowIPSpoofing bool `yaml:"allowIPSpoofing"` - bittorrent.RequestSanitizer `yaml:",inline"` + AllowIPSpoofing bool `yaml:"allow_ip_spoofing"` + MaxNumWant uint32 `yaml:"max_numwant"` + DefaultNumWant uint32 `yaml:"default_numwant"` + MaxScrapeInfoHashes uint32 `yaml:"max_scrape_infohashes"` } // ParseAnnounce parses an AnnounceRequest from a UDP request. @@ -117,7 +119,7 @@ func ParseAnnounce(r Request, v6 bool, opts ParseOptions) (*bittorrent.AnnounceR Params: params, } - if err := opts.SanitizeAnnounce(request); err != nil { + if err := bittorrent.SanitizeAnnounce(request, opts.MaxNumWant, opts.DefaultNumWant); err != nil { return nil, err } @@ -208,7 +210,7 @@ func ParseScrape(r Request, opts ParseOptions) (*bittorrent.ScrapeRequest, error // Sanitize the request. request := &bittorrent.ScrapeRequest{InfoHashes: infohashes} - if err := opts.SanitizeScrape(request); err != nil { + if err := bittorrent.SanitizeScrape(request, opts.MaxScrapeInfoHashes); err != nil { return nil, err } From 1a0b5c56a6a53ac741a94446c92b66f3730c01c0 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Tue, 17 Oct 2017 22:06:03 -0400 Subject: [PATCH 8/9] frontend/http: disambiguate NumWantProvided --- frontend/http/parser.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/http/parser.go b/frontend/http/parser.go index c463936..ef7b207 100644 --- a/frontend/http/parser.go +++ b/frontend/http/parser.go @@ -88,8 +88,9 @@ func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequ if err != nil && err != bittorrent.ErrKeyNotFound { return nil, bittorrent.ClientError("failed to parse parameter: numwant") } - request.NumWant = uint32(numwant) + // If there were no errors, the user actually provided the numwant. request.NumWantProvided = err == nil + request.NumWant = uint32(numwant) // Parse the port where the client is listening. port, err := qp.Uint64("port") From df0de9433783305c4ae82a96a3a57d25260960b0 Mon Sep 17 00:00:00 2001 From: Jimmy Zelinskie Date: Wed, 18 Oct 2017 11:51:19 -0400 Subject: [PATCH 9/9] frontend/http: bandwidth are in bytes not pieces --- frontend/http/parser.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/frontend/http/parser.go b/frontend/http/parser.go index ef7b207..3053d82 100644 --- a/frontend/http/parser.go +++ b/frontend/http/parser.go @@ -65,19 +65,19 @@ func ParseAnnounce(r *http.Request, opts ParseOptions) (*bittorrent.AnnounceRequ } request.Peer.ID = bittorrent.PeerIDFromString(peerID) - // Determine the number of remaining pieces for the client. + // Determine the number of remaining bytes for the client. request.Left, err = qp.Uint64("left") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: left") } - // Determine the number of pieces downloaded by the client. + // Determine the number of bytes downloaded by the client. request.Downloaded, err = qp.Uint64("downloaded") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: downloaded") } - // Determine the number of pieces shared by the client. + // Determine the number of bytes shared by the client. request.Uploaded, err = qp.Uint64("uploaded") if err != nil { return nil, bittorrent.ClientError("failed to parse parameter: uploaded")