upgrade dependencies

adjust transaction summary fields

.travis.yml

@@ -1,8 +1,8 @@
 os: linux
-dist: xenial
+dist: bionic
 language: go
 go:
-  - 1.15.x
+  - 1.17.x
 
 env:
   global:

blobex/blobex.pb.go

@@ -0,0 +1,765 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: blobex.proto
+
+package blobex
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+import (
+	context "golang.org/x/net/context"
+	grpc "google.golang.org/grpc"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type Error struct {
+	// should we enum the error codes?
+	Code                 uint32   `protobuf:"varint,1,opt,name=code,proto3" json:"code,omitempty"`
+	Message              string   `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Error) Reset()         { *m = Error{} }
+func (m *Error) String() string { return proto.CompactTextString(m) }
+func (*Error) ProtoMessage()    {}
+func (*Error) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{0}
+}
+
+func (m *Error) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Error.Unmarshal(m, b)
+}
+func (m *Error) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Error.Marshal(b, m, deterministic)
+}
+func (m *Error) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Error.Merge(m, src)
+}
+func (m *Error) XXX_Size() int {
+	return xxx_messageInfo_Error.Size(m)
+}
+func (m *Error) XXX_DiscardUnknown() {
+	xxx_messageInfo_Error.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Error proto.InternalMessageInfo
+
+func (m *Error) GetCode() uint32 {
+	if m != nil {
+		return m.Code
+	}
+	return 0
+}
+
+func (m *Error) GetMessage() string {
+	if m != nil {
+		return m.Message
+	}
+	return ""
+}
+
+// how much does the host charge per kb at the moment
+type PriceCheckRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *PriceCheckRequest) Reset()         { *m = PriceCheckRequest{} }
+func (m *PriceCheckRequest) String() string { return proto.CompactTextString(m) }
+func (*PriceCheckRequest) ProtoMessage()    {}
+func (*PriceCheckRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{1}
+}
+
+func (m *PriceCheckRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PriceCheckRequest.Unmarshal(m, b)
+}
+func (m *PriceCheckRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PriceCheckRequest.Marshal(b, m, deterministic)
+}
+func (m *PriceCheckRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PriceCheckRequest.Merge(m, src)
+}
+func (m *PriceCheckRequest) XXX_Size() int {
+	return xxx_messageInfo_PriceCheckRequest.Size(m)
+}
+func (m *PriceCheckRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_PriceCheckRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PriceCheckRequest proto.InternalMessageInfo
+
+type PriceCheckResponse struct {
+	Error                *Error   `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	DeweysPerKB          uint64   `protobuf:"varint,2,opt,name=deweysPerKB,proto3" json:"deweysPerKB,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *PriceCheckResponse) Reset()         { *m = PriceCheckResponse{} }
+func (m *PriceCheckResponse) String() string { return proto.CompactTextString(m) }
+func (*PriceCheckResponse) ProtoMessage()    {}
+func (*PriceCheckResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{2}
+}
+
+func (m *PriceCheckResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PriceCheckResponse.Unmarshal(m, b)
+}
+func (m *PriceCheckResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PriceCheckResponse.Marshal(b, m, deterministic)
+}
+func (m *PriceCheckResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PriceCheckResponse.Merge(m, src)
+}
+func (m *PriceCheckResponse) XXX_Size() int {
+	return xxx_messageInfo_PriceCheckResponse.Size(m)
+}
+func (m *PriceCheckResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_PriceCheckResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PriceCheckResponse proto.InternalMessageInfo
+
+func (m *PriceCheckResponse) GetError() *Error {
+	if m != nil {
+		return m.Error
+	}
+	return nil
+}
+
+func (m *PriceCheckResponse) GetDeweysPerKB() uint64 {
+	if m != nil {
+		return m.DeweysPerKB
+	}
+	return 0
+}
+
+// are any of the hashs available for download, or are any of the hashes desired for upload
+// NOTE: if any hashes are stream hashes, and the server has the manifest but not all the content
+// blobs, the server may reply that it needs extra blobs that were not in the original request
+type HashesRequest struct {
+	Hashes               []string `protobuf:"bytes,1,rep,name=hashes,proto3" json:"hashes,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HashesRequest) Reset()         { *m = HashesRequest{} }
+func (m *HashesRequest) String() string { return proto.CompactTextString(m) }
+func (*HashesRequest) ProtoMessage()    {}
+func (*HashesRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{3}
+}
+
+func (m *HashesRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HashesRequest.Unmarshal(m, b)
+}
+func (m *HashesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HashesRequest.Marshal(b, m, deterministic)
+}
+func (m *HashesRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HashesRequest.Merge(m, src)
+}
+func (m *HashesRequest) XXX_Size() int {
+	return xxx_messageInfo_HashesRequest.Size(m)
+}
+func (m *HashesRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_HashesRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HashesRequest proto.InternalMessageInfo
+
+func (m *HashesRequest) GetHashes() []string {
+	if m != nil {
+		return m.Hashes
+	}
+	return nil
+}
+
+type HashesResponse struct {
+	Error                *Error          `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	Hashes               map[string]bool `protobuf:"bytes,2,rep,name=hashes,proto3" json:"hashes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"varint,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}        `json:"-"`
+	XXX_unrecognized     []byte          `json:"-"`
+	XXX_sizecache        int32           `json:"-"`
+}
+
+func (m *HashesResponse) Reset()         { *m = HashesResponse{} }
+func (m *HashesResponse) String() string { return proto.CompactTextString(m) }
+func (*HashesResponse) ProtoMessage()    {}
+func (*HashesResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{4}
+}
+
+func (m *HashesResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HashesResponse.Unmarshal(m, b)
+}
+func (m *HashesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HashesResponse.Marshal(b, m, deterministic)
+}
+func (m *HashesResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HashesResponse.Merge(m, src)
+}
+func (m *HashesResponse) XXX_Size() int {
+	return xxx_messageInfo_HashesResponse.Size(m)
+}
+func (m *HashesResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_HashesResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HashesResponse proto.InternalMessageInfo
+
+func (m *HashesResponse) GetError() *Error {
+	if m != nil {
+		return m.Error
+	}
+	return nil
+}
+
+func (m *HashesResponse) GetHashes() map[string]bool {
+	if m != nil {
+		return m.Hashes
+	}
+	return nil
+}
+
+// download the hash
+type DownloadRequest struct {
+	Hash                 string   `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DownloadRequest) Reset()         { *m = DownloadRequest{} }
+func (m *DownloadRequest) String() string { return proto.CompactTextString(m) }
+func (*DownloadRequest) ProtoMessage()    {}
+func (*DownloadRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{5}
+}
+
+func (m *DownloadRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DownloadRequest.Unmarshal(m, b)
+}
+func (m *DownloadRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DownloadRequest.Marshal(b, m, deterministic)
+}
+func (m *DownloadRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DownloadRequest.Merge(m, src)
+}
+func (m *DownloadRequest) XXX_Size() int {
+	return xxx_messageInfo_DownloadRequest.Size(m)
+}
+func (m *DownloadRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_DownloadRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DownloadRequest proto.InternalMessageInfo
+
+func (m *DownloadRequest) GetHash() string {
+	if m != nil {
+		return m.Hash
+	}
+	return ""
+}
+
+type DownloadResponse struct {
+	Error                *Error   `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	Hash                 string   `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"`
+	Blob                 []byte   `protobuf:"bytes,3,opt,name=blob,proto3" json:"blob,omitempty"`
+	Address              string   `protobuf:"bytes,4,opt,name=address,proto3" json:"address,omitempty"`
+	Price                uint64   `protobuf:"varint,5,opt,name=price,proto3" json:"price,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DownloadResponse) Reset()         { *m = DownloadResponse{} }
+func (m *DownloadResponse) String() string { return proto.CompactTextString(m) }
+func (*DownloadResponse) ProtoMessage()    {}
+func (*DownloadResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{6}
+}
+
+func (m *DownloadResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DownloadResponse.Unmarshal(m, b)
+}
+func (m *DownloadResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DownloadResponse.Marshal(b, m, deterministic)
+}
+func (m *DownloadResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DownloadResponse.Merge(m, src)
+}
+func (m *DownloadResponse) XXX_Size() int {
+	return xxx_messageInfo_DownloadResponse.Size(m)
+}
+func (m *DownloadResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_DownloadResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DownloadResponse proto.InternalMessageInfo
+
+func (m *DownloadResponse) GetError() *Error {
+	if m != nil {
+		return m.Error
+	}
+	return nil
+}
+
+func (m *DownloadResponse) GetHash() string {
+	if m != nil {
+		return m.Hash
+	}
+	return ""
+}
+
+func (m *DownloadResponse) GetBlob() []byte {
+	if m != nil {
+		return m.Blob
+	}
+	return nil
+}
+
+func (m *DownloadResponse) GetAddress() string {
+	if m != nil {
+		return m.Address
+	}
+	return ""
+}
+
+func (m *DownloadResponse) GetPrice() uint64 {
+	if m != nil {
+		return m.Price
+	}
+	return 0
+}
+
+// upload the hash
+type UploadRequest struct {
+	Hash                 string   `protobuf:"bytes,1,opt,name=hash,proto3" json:"hash,omitempty"`
+	Blob                 []byte   `protobuf:"bytes,2,opt,name=blob,proto3" json:"blob,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *UploadRequest) Reset()         { *m = UploadRequest{} }
+func (m *UploadRequest) String() string { return proto.CompactTextString(m) }
+func (*UploadRequest) ProtoMessage()    {}
+func (*UploadRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{7}
+}
+
+func (m *UploadRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_UploadRequest.Unmarshal(m, b)
+}
+func (m *UploadRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_UploadRequest.Marshal(b, m, deterministic)
+}
+func (m *UploadRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_UploadRequest.Merge(m, src)
+}
+func (m *UploadRequest) XXX_Size() int {
+	return xxx_messageInfo_UploadRequest.Size(m)
+}
+func (m *UploadRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_UploadRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UploadRequest proto.InternalMessageInfo
+
+func (m *UploadRequest) GetHash() string {
+	if m != nil {
+		return m.Hash
+	}
+	return ""
+}
+
+func (m *UploadRequest) GetBlob() []byte {
+	if m != nil {
+		return m.Blob
+	}
+	return nil
+}
+
+type UploadResponse struct {
+	Error                *Error   `protobuf:"bytes,1,opt,name=error,proto3" json:"error,omitempty"`
+	Hash                 string   `protobuf:"bytes,2,opt,name=hash,proto3" json:"hash,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *UploadResponse) Reset()         { *m = UploadResponse{} }
+func (m *UploadResponse) String() string { return proto.CompactTextString(m) }
+func (*UploadResponse) ProtoMessage()    {}
+func (*UploadResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_183aee39e18f30c9, []int{8}
+}
+
+func (m *UploadResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_UploadResponse.Unmarshal(m, b)
+}
+func (m *UploadResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_UploadResponse.Marshal(b, m, deterministic)
+}
+func (m *UploadResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_UploadResponse.Merge(m, src)
+}
+func (m *UploadResponse) XXX_Size() int {
+	return xxx_messageInfo_UploadResponse.Size(m)
+}
+func (m *UploadResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_UploadResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UploadResponse proto.InternalMessageInfo
+
+func (m *UploadResponse) GetError() *Error {
+	if m != nil {
+		return m.Error
+	}
+	return nil
+}
+
+func (m *UploadResponse) GetHash() string {
+	if m != nil {
+		return m.Hash
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*Error)(nil), "blobex.Error")
+	proto.RegisterType((*PriceCheckRequest)(nil), "blobex.PriceCheckRequest")
+	proto.RegisterType((*PriceCheckResponse)(nil), "blobex.PriceCheckResponse")
+	proto.RegisterType((*HashesRequest)(nil), "blobex.HashesRequest")
+	proto.RegisterType((*HashesResponse)(nil), "blobex.HashesResponse")
+	proto.RegisterMapType((map[string]bool)(nil), "blobex.HashesResponse.HashesEntry")
+	proto.RegisterType((*DownloadRequest)(nil), "blobex.DownloadRequest")
+	proto.RegisterType((*DownloadResponse)(nil), "blobex.DownloadResponse")
+	proto.RegisterType((*UploadRequest)(nil), "blobex.UploadRequest")
+	proto.RegisterType((*UploadResponse)(nil), "blobex.UploadResponse")
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// BlobExchangeClient is the client API for BlobExchange service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type BlobExchangeClient interface {
+	PriceCheck(ctx context.Context, in *PriceCheckRequest, opts ...grpc.CallOption) (*PriceCheckResponse, error)
+	DownloadCheck(ctx context.Context, in *HashesRequest, opts ...grpc.CallOption) (*HashesResponse, error)
+	Download(ctx context.Context, opts ...grpc.CallOption) (BlobExchange_DownloadClient, error)
+	UploadCheck(ctx context.Context, in *HashesRequest, opts ...grpc.CallOption) (*HashesResponse, error)
+	Upload(ctx context.Context, opts ...grpc.CallOption) (BlobExchange_UploadClient, error)
+}
+
+type blobExchangeClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewBlobExchangeClient(cc *grpc.ClientConn) BlobExchangeClient {
+	return &blobExchangeClient{cc}
+}
+
+func (c *blobExchangeClient) PriceCheck(ctx context.Context, in *PriceCheckRequest, opts ...grpc.CallOption) (*PriceCheckResponse, error) {
+	out := new(PriceCheckResponse)
+	err := c.cc.Invoke(ctx, "/blobex.BlobExchange/PriceCheck", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *blobExchangeClient) DownloadCheck(ctx context.Context, in *HashesRequest, opts ...grpc.CallOption) (*HashesResponse, error) {
+	out := new(HashesResponse)
+	err := c.cc.Invoke(ctx, "/blobex.BlobExchange/DownloadCheck", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *blobExchangeClient) Download(ctx context.Context, opts ...grpc.CallOption) (BlobExchange_DownloadClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_BlobExchange_serviceDesc.Streams[0], "/blobex.BlobExchange/Download", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &blobExchangeDownloadClient{stream}
+	return x, nil
+}
+
+type BlobExchange_DownloadClient interface {
+	Send(*DownloadRequest) error
+	Recv() (*DownloadResponse, error)
+	grpc.ClientStream
+}
+
+type blobExchangeDownloadClient struct {
+	grpc.ClientStream
+}
+
+func (x *blobExchangeDownloadClient) Send(m *DownloadRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *blobExchangeDownloadClient) Recv() (*DownloadResponse, error) {
+	m := new(DownloadResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func (c *blobExchangeClient) UploadCheck(ctx context.Context, in *HashesRequest, opts ...grpc.CallOption) (*HashesResponse, error) {
+	out := new(HashesResponse)
+	err := c.cc.Invoke(ctx, "/blobex.BlobExchange/UploadCheck", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *blobExchangeClient) Upload(ctx context.Context, opts ...grpc.CallOption) (BlobExchange_UploadClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_BlobExchange_serviceDesc.Streams[1], "/blobex.BlobExchange/Upload", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &blobExchangeUploadClient{stream}
+	return x, nil
+}
+
+type BlobExchange_UploadClient interface {
+	Send(*UploadRequest) error
+	Recv() (*UploadResponse, error)
+	grpc.ClientStream
+}
+
+type blobExchangeUploadClient struct {
+	grpc.ClientStream
+}
+
+func (x *blobExchangeUploadClient) Send(m *UploadRequest) error {
+	return x.ClientStream.SendMsg(m)
+}
+
+func (x *blobExchangeUploadClient) Recv() (*UploadResponse, error) {
+	m := new(UploadResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// BlobExchangeServer is the server API for BlobExchange service.
+type BlobExchangeServer interface {
+	PriceCheck(context.Context, *PriceCheckRequest) (*PriceCheckResponse, error)
+	DownloadCheck(context.Context, *HashesRequest) (*HashesResponse, error)
+	Download(BlobExchange_DownloadServer) error
+	UploadCheck(context.Context, *HashesRequest) (*HashesResponse, error)
+	Upload(BlobExchange_UploadServer) error
+}
+
+func RegisterBlobExchangeServer(s *grpc.Server, srv BlobExchangeServer) {
+	s.RegisterService(&_BlobExchange_serviceDesc, srv)
+}
+
+func _BlobExchange_PriceCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(PriceCheckRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BlobExchangeServer).PriceCheck(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/blobex.BlobExchange/PriceCheck",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BlobExchangeServer).PriceCheck(ctx, req.(*PriceCheckRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _BlobExchange_DownloadCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HashesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BlobExchangeServer).DownloadCheck(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/blobex.BlobExchange/DownloadCheck",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BlobExchangeServer).DownloadCheck(ctx, req.(*HashesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _BlobExchange_Download_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BlobExchangeServer).Download(&blobExchangeDownloadServer{stream})
+}
+
+type BlobExchange_DownloadServer interface {
+	Send(*DownloadResponse) error
+	Recv() (*DownloadRequest, error)
+	grpc.ServerStream
+}
+
+type blobExchangeDownloadServer struct {
+	grpc.ServerStream
+}
+
+func (x *blobExchangeDownloadServer) Send(m *DownloadResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *blobExchangeDownloadServer) Recv() (*DownloadRequest, error) {
+	m := new(DownloadRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+func _BlobExchange_UploadCheck_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(HashesRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(BlobExchangeServer).UploadCheck(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/blobex.BlobExchange/UploadCheck",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(BlobExchangeServer).UploadCheck(ctx, req.(*HashesRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _BlobExchange_Upload_Handler(srv interface{}, stream grpc.ServerStream) error {
+	return srv.(BlobExchangeServer).Upload(&blobExchangeUploadServer{stream})
+}
+
+type BlobExchange_UploadServer interface {
+	Send(*UploadResponse) error
+	Recv() (*UploadRequest, error)
+	grpc.ServerStream
+}
+
+type blobExchangeUploadServer struct {
+	grpc.ServerStream
+}
+
+func (x *blobExchangeUploadServer) Send(m *UploadResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
+func (x *blobExchangeUploadServer) Recv() (*UploadRequest, error) {
+	m := new(UploadRequest)
+	if err := x.ServerStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+var _BlobExchange_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "blobex.BlobExchange",
+	HandlerType: (*BlobExchangeServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "PriceCheck",
+			Handler:    _BlobExchange_PriceCheck_Handler,
+		},
+		{
+			MethodName: "DownloadCheck",
+			Handler:    _BlobExchange_DownloadCheck_Handler,
+		},
+		{
+			MethodName: "UploadCheck",
+			Handler:    _BlobExchange_UploadCheck_Handler,
+		},
+	},
+	Streams: []grpc.StreamDesc{
+		{
+			StreamName:    "Download",
+			Handler:       _BlobExchange_Download_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+		{
+			StreamName:    "Upload",
+			Handler:       _BlobExchange_Upload_Handler,
+			ServerStreams: true,
+			ClientStreams: true,
+		},
+	},
+	Metadata: "blobex.proto",
+}
+
+func init() { proto.RegisterFile("blobex.proto", fileDescriptor_183aee39e18f30c9) }
+
+var fileDescriptor_183aee39e18f30c9 = []byte{
+	// 449 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x54, 0xcd, 0x6e, 0x13, 0x31,
+	0x10, 0xae, 0x37, 0x3f, 0x34, 0xb3, 0xd9, 0x52, 0x06, 0x28, 0x26, 0xa7, 0x95, 0x11, 0x62, 0x4f,
+	0x15, 0x0a, 0x42, 0x40, 0x25, 0x10, 0x6a, 0x89, 0x04, 0xe2, 0x52, 0x59, 0xe2, 0xc4, 0x69, 0x93,
+	0x1d, 0x35, 0xa8, 0xcb, 0x7a, 0xb1, 0x53, 0xda, 0x3c, 0x05, 0x2f, 0xc2, 0x1b, 0xf1, 0x32, 0xc8,
+	0xf6, 0xba, 0x9b, 0xb4, 0x95, 0x50, 0xe8, 0x6d, 0xbe, 0xf1, 0xcc, 0xf7, 0x8d, 0xfd, 0x8d, 0x0c,
+	0xc3, 0x69, 0xa9, 0xa6, 0x74, 0xb1, 0x5f, 0x6b, 0xb5, 0x50, 0xd8, 0xf7, 0x48, 0xbc, 0x84, 0xde,
+	0x44, 0x6b, 0xa5, 0x11, 0xa1, 0x3b, 0x53, 0x05, 0x71, 0x96, 0xb2, 0x2c, 0x91, 0x2e, 0x46, 0x0e,
+	0x77, 0xbe, 0x93, 0x31, 0xf9, 0x09, 0xf1, 0x28, 0x65, 0xd9, 0x40, 0x06, 0x28, 0xee, 0xc3, 0xbd,
+	0x63, 0xfd, 0x6d, 0x46, 0x47, 0x73, 0x9a, 0x9d, 0x4a, 0xfa, 0x71, 0x46, 0x66, 0x21, 0xbe, 0x02,
+	0xae, 0x26, 0x4d, 0xad, 0x2a, 0x43, 0xf8, 0x04, 0x7a, 0x64, 0x15, 0x1c, 0x73, 0x3c, 0x4e, 0xf6,
+	0x9b, 0x39, 0x9c, 0xac, 0xf4, 0x67, 0x98, 0x42, 0x5c, 0xd0, 0x39, 0x2d, 0xcd, 0x31, 0xe9, 0xcf,
+	0x87, 0x4e, 0xad, 0x2b, 0x57, 0x53, 0xe2, 0x19, 0x24, 0x1f, 0x73, 0x33, 0x27, 0xd3, 0xa8, 0xe1,
+	0x1e, 0xf4, 0xe7, 0x2e, 0xc1, 0x59, 0xda, 0xc9, 0x06, 0xb2, 0x41, 0xe2, 0x37, 0x83, 0x9d, 0x50,
+	0xb9, 0xc9, 0x08, 0x07, 0x97, 0x7c, 0x51, 0xda, 0xc9, 0xe2, 0xb1, 0x08, 0x55, 0xeb, 0x64, 0x0d,
+	0x9c, 0x54, 0x0b, 0xbd, 0x0c, 0x9a, 0xa3, 0x37, 0x10, 0xaf, 0xa4, 0x71, 0x17, 0x3a, 0xa7, 0xb4,
+	0x74, 0x6a, 0x03, 0x69, 0x43, 0x7c, 0x00, 0xbd, 0x9f, 0x79, 0x79, 0xe6, 0xdf, 0x71, 0x5b, 0x7a,
+	0x70, 0x10, 0xbd, 0x66, 0xe2, 0x29, 0xdc, 0xfd, 0xa0, 0xce, 0xab, 0x52, 0xe5, 0x45, 0xb8, 0x19,
+	0x42, 0xd7, 0xf2, 0x36, 0xfd, 0x2e, 0x16, 0xbf, 0x18, 0xec, 0xb6, 0x75, 0x9b, 0xdc, 0x2b, 0xb0,
+	0x45, 0x2d, 0x9b, 0xcd, 0xd9, 0x52, 0xde, 0x49, 0x59, 0x36, 0x94, 0x2e, 0xb6, 0x66, 0xe7, 0x45,
+	0xa1, 0xc9, 0x18, 0xde, 0xf5, 0x66, 0x37, 0xd0, 0x0e, 0x5f, 0x5b, 0x5f, 0x79, 0xcf, 0xd9, 0xe2,
+	0x81, 0x78, 0x05, 0xc9, 0x97, 0xfa, 0x1f, 0x63, 0x5f, 0x0a, 0x45, 0xad, 0x90, 0xf8, 0x04, 0x3b,
+	0xa1, 0xf1, 0x96, 0xf7, 0x18, 0xff, 0x89, 0x60, 0x78, 0x58, 0xaa, 0xe9, 0xe4, 0x62, 0x36, 0xcf,
+	0xab, 0x13, 0xc2, 0x09, 0x40, 0xbb, 0x82, 0xf8, 0x38, 0x10, 0x5d, 0xdb, 0xd5, 0xd1, 0xe8, 0xa6,
+	0x23, 0x3f, 0x8e, 0xd8, 0xc2, 0xf7, 0x90, 0x84, 0xc7, 0xf6, 0x4c, 0x0f, 0xaf, 0x2e, 0x83, 0x67,
+	0xd9, 0xbb, 0x79, 0x47, 0xc4, 0x16, 0x1e, 0xc1, 0x76, 0x60, 0xc0, 0x47, 0xa1, 0xea, 0x8a, 0xd1,
+	0x23, 0x7e, 0xfd, 0x20, 0x10, 0x64, 0xec, 0x39, 0xc3, 0x77, 0x10, 0xfb, 0x97, 0xfa, 0xcf, 0x21,
+	0xde, 0x42, 0xdf, 0xf7, 0xb7, 0xad, 0x6b, 0x96, 0xb5, 0xad, 0xeb, 0x86, 0x78, 0xf9, 0x69, 0xdf,
+	0x7d, 0x15, 0x2f, 0xfe, 0x06, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x4a, 0x7e, 0x89, 0x3a, 0x04, 0x00,
+	0x00,
+}

blobex/blobex.proto

@@ -0,0 +1,87 @@
+syntax = "proto3";
+
+package blobex;
+
+/*
+
+## Errors
+- invalid request
+- banned for nonpayment
+- banned for uploading unwanted blobs
+- blob not wanted
+- blob not available
+- not accepting blobs
+
+## Considerations
+
+- there are two requests to upload a blob. how to reduce that to one?
+  - UploadCheck checks for many hashes at once. if you're just uploading one or a few, just do it and handle the error
+
+- how to avoid receiving the whole blob and then determining the blob is not wanted? may not ever be possible
+
+- is avail check necessary? just request what you want for download
+  - maybe you want to check multiple blobs at once?
+
+- how to check for wanted blobs from stream hash?
+
+- prices should be set by hosts, since they are offering blobs for download and have the best information on prices. but request is initiated by client. how do we make sure clients are not overpaying without making them make two requests for each blob?
+
+- should we have another request to submit proof of payment? or at least a txid?
+
+*/
+
+service BlobExchange {
+    rpc PriceCheck(PriceCheckRequest) returns (PriceCheckResponse) {}
+    rpc DownloadCheck(HashesRequest) returns (HashesResponse) {}
+    rpc Download(stream DownloadRequest) returns (stream DownloadResponse) {}
+    rpc UploadCheck(HashesRequest) returns (HashesResponse) {}
+    rpc Upload(stream UploadRequest) returns (stream UploadResponse) {}
+}
+
+message Error {
+    // should we enum the error codes?
+    uint32 code = 1;
+    string message = 2;
+}
+
+// how much does the host charge per kb at the moment
+message PriceCheckRequest {
+}
+message PriceCheckResponse{
+    Error error = 1;
+    uint64 deweysPerKB = 2;
+}
+
+// are any of the hashs available for download, or are any of the hashes desired for upload
+// NOTE: if any hashes are stream hashes, and the server has the manifest but not all the content
+// blobs, the server may reply that it needs extra blobs that were not in the original request
+message HashesRequest {
+    repeated string hashes = 1;
+}
+message HashesResponse {
+    Error error = 1;
+    map<string, bool> hashes = 2;
+}
+
+// download the hash
+message DownloadRequest {
+    string hash = 1;
+}
+message DownloadResponse {
+    Error error = 1;
+    string hash = 2;
+    bytes blob = 3;
+    string address = 4;  // address where payment for data download should be sent
+    uint64 price = 5;    // price of the data in this blob
+}
+
+// upload the hash
+message UploadRequest {
+    string hash = 1;
+    bytes blob = 2;
+}
+message UploadResponse {
+    Error error = 1;
+    string hash = 2;
+}
+

blobex/proto.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+set -euo pipefail
+#set -x
+
+version_gte() {
+  [ "$1" = "$(echo -e "$1\n$2" | sort -V | tail -n1)" ]
+}
+
+DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+
+
+hash protoc 2>/dev/null || { echo >&2 -e 'error: protoc binary not found\nDownload it from https://github.com/google/protobuf/releases and put it in your path.\nMake sure you get the one starting with `protoc`, not `protobuf`.'; exit 1; }
+
+
+PROTOC="$(which protoc)"
+VERSION="$($PROTOC --version | cut -d' ' -f2)"
+MIN_VERSION="3.0"
+
+version_gte "$VERSION" "$MIN_VERSION" || { echo >&2 "error: protoc version must be >= $MIN_VERSION (your $PROTOC is $VERSION)"; exit 1; }
+
+
+hash protoc-gen-go 2>/dev/null || go get -u github.com/golang/protobuf/protoc-gen-go
+hash protoc-gen-go 2>/dev/null || { echo >&2 'error: Make sure $GOPATH/bin is in your $PATH'; exit 1; }
+
+
+find . -type f -iname '*.pb.go' -delete
+
+protoc --proto_path=. blobex.proto --go_out=plugins=grpc:.

blobex/server.go

@@ -0,0 +1,49 @@
+package blobex
+
+import (
+	"fmt"
+	"net"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+)
+
+type Server struct {
+	pricePerKB uint64
+}
+
+func ListenAndServe(port int) (*grpc.Server, error) {
+	listener, err := net.Listen("tcp", fmt.Sprintf(":%d", port))
+	if err != nil {
+		return nil, errors.Prefix("failed to listen", err)
+	}
+	grpcServer := grpc.NewServer()
+	RegisterBlobExchangeServer(grpcServer, &Server{})
+	// determine whether to use TLS
+	err = grpcServer.Serve(listener)
+	return grpcServer, err
+}
+
+func (s *Server) PriceCheck(ctx context.Context, r *PriceCheckRequest) (*PriceCheckResponse, error) {
+	return &PriceCheckResponse{
+		DeweysPerKB: s.pricePerKB,
+	}, nil
+}
+
+func (s *Server) DownloadCheck(context.Context, *HashesRequest) (*HashesResponse, error) {
+	return nil, nil
+}
+
+func (s *Server) Download(BlobExchange_DownloadServer) error {
+	return nil
+}
+
+func (s *Server) UploadCheck(context.Context, *HashesRequest) (*HashesResponse, error) {
+	return nil, nil
+}
+
+func (s *Server) Upload(BlobExchange_UploadServer) error {
+	return nil
+}

claim/script.go

@@ -1,84 +0,0 @@
-package claim
-
-import (
-	"encoding/hex"
-
-	"github.com/cockroachdb/errors"
-	"github.com/lbryio/lbcd/txscript"
-	"github.com/lbryio/lbcutil"
-)
-
-func ClaimSupportPayoutScript(name, claimid string, address lbcutil.Address) ([]byte, error) {
-	//OP_SUPPORT_CLAIM <name> <claimid> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
-
-	pkscript, err := txscript.PayToAddrScript(address)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	bytes, err := hex.DecodeString(claimid)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	return txscript.NewScriptBuilder().
-		AddOp(txscript.OP_SUPPORTCLAIM). //OP_SUPPORT_CLAIM
-		AddData([]byte(name)).           //<name>
-		AddData(rev(bytes)).             //<claimid>
-		AddOp(txscript.OP_2DROP).        //OP_2DROP
-		AddOp(txscript.OP_DROP).         //OP_DROP
-		AddOps(pkscript).                //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
-		Script()
-
-}
-
-func ClaimNamePayoutScript(name string, value []byte, address lbcutil.Address) ([]byte, error) {
-	//OP_CLAIM_NAME <name> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
-
-	pkscript, err := txscript.PayToAddrScript(address)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	return txscript.NewScriptBuilder().
-		AddOp(txscript.OP_CLAIMNAME). //OP_CLAIMNAME
-		AddData([]byte(name)).        //<name>
-		AddData(value).               //<value>
-		AddOp(txscript.OP_2DROP).     //OP_2DROP
-		AddOp(txscript.OP_DROP).      //OP_DROP
-		AddOps(pkscript).             //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
-		Script()
-}
-
-func UpdateClaimPayoutScript(name, claimid string, value []byte, address lbcutil.Address) ([]byte, error) {
-	//OP_UPDATE_CLAIM <name> <claimid> <value> OP_2DROP OP_DROP OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
-
-	pkscript, err := txscript.PayToAddrScript(address)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	bytes, err := hex.DecodeString(claimid)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	return txscript.NewScriptBuilder().
-		AddOp(txscript.OP_UPDATECLAIM). //OP_UPDATE_CLAIM
-		AddData([]byte(name)).          //<name>
-		AddData(rev(bytes)).            //<claimid>
-		AddData(value).                 //<value>
-		AddOp(txscript.OP_2DROP).       //OP_2DROP
-		AddOp(txscript.OP_DROP).        //OP_DROP
-		AddOps(pkscript).               //OP_DUP OP_HASH160 <address> OP_EQUALVERIFY OP_CHECKSIG
-		Script()
-}
-
-// rev reverses a byte slice. useful for switching endian-ness
-func rev(b []byte) []byte {
-	r := make([]byte, len(b))
-	for left, right := 0, len(b)-1; left < right; left, right = left+1, right-1 {
-		r[left], r[right] = b[right], b[left]
-	}
-	return r
-}

dht/bits/bitmap.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/cockroachdb/errors"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 
 	"github.com/lyoshenka/bencode"
 )
@@ -254,7 +254,7 @@ func (b *Bitmap) UnmarshalBencode(encoded []byte) error {
 		return err
 	}
 	if len(str) != NumBytes {
-		return errors.WithStack(errors.New("invalid bitmap length"))
+		return errors.Err("invalid bitmap length")
 	}
 	copy(b[:], str)
 	return nil
@@ -265,7 +265,7 @@ func FromBytes(data []byte) (Bitmap, error) {
 	var bmp Bitmap
 
 	if len(data) != len(bmp) {
-		return bmp, errors.WithStack(errors.Newf("invalid bitmap of length %d", len(data)))
+		return bmp, errors.Err("invalid bitmap of length %d", len(data))
 	}
 
 	copy(bmp[:], data)
@@ -303,7 +303,7 @@ func FromStringP(data string) Bitmap {
 func FromHex(hexStr string) (Bitmap, error) {
 	decoded, err := hex.DecodeString(hexStr)
 	if err != nil {
-		return Bitmap{}, errors.WithStack(err)
+		return Bitmap{}, errors.Err(err)
 	}
 	return FromBytes(decoded)
 }

dht/bits/range.go

@@ -3,7 +3,7 @@ package bits
 import (
 	"math/big"
 
-	"github.com/cockroachdb/errors"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 )
 
 // Range has a start and end
@@ -24,7 +24,7 @@ func MaxRange() Range {
 // the first interval always starts at the beginning of the range, and the last interval always ends at the end
 func (r Range) IntervalP(n, num int) Range {
 	if num < 1 || n < 1 || n > num {
-		panic(errors.WithStack(errors.Newf("invalid interval %d of %d", n, num)))
+		panic(errors.Err("invalid interval %d of %d", n, num))
 	}
 
 	start := r.intervalStart(n, num)

dht/bootstrap.go

@@ -6,7 +6,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 const (

dht/bootstrap_test.go

@@ -4,7 +4,7 @@ import (
 	"net"
 	"testing"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 func TestBootstrapPing(t *testing.T) {

dht/config.go

@@ -4,7 +4,7 @@ import (
 	"strconv"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 const (

dht/contact.go

@@ -7,9 +7,9 @@ import (
 	"sort"
 	"strconv"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 
-	"github.com/cockroachdb/errors"
 	"github.com/lyoshenka/bencode"
 )
 
@@ -43,7 +43,7 @@ func (c Contact) String() string {
 }
 
 func (c Contact) MarshalJSON() ([]byte, error) {
-	b, err := json.Marshal(&struct {
+	return json.Marshal(&struct {
 		ID       string
 		IP       string
 		Port     int
@@ -54,17 +54,16 @@ func (c Contact) MarshalJSON() ([]byte, error) {
 		Port:     c.Port,
 		PeerPort: c.PeerPort,
 	})
-	return b, errors.Wrap(err, "")
 }
 
 // MarshalCompact returns a compact byteslice representation of the contact
 // NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
 func (c Contact) MarshalCompact() ([]byte, error) {
 	if c.IP.To4() == nil {
-		return nil, errors.Wrap(errors.New("ip not set"), "")
+		return nil, errors.Err("ip not set")
 	}
 	if c.PeerPort < 0 || c.PeerPort > 65535 {
-		return nil, errors.Wrap(errors.New("invalid port"), "")
+		return nil, errors.Err("invalid port")
 	}
 
 	var buf bytes.Buffer
@@ -74,7 +73,7 @@ func (c Contact) MarshalCompact() ([]byte, error) {
 	buf.Write(c.ID[:])
 
 	if buf.Len() != compactNodeInfoLength {
-		return nil, errors.Wrap(errors.New("i dont know how this happened"), "")
+		return nil, errors.Err("i dont know how this happened")
 	}
 
 	return buf.Bytes(), nil
@@ -84,7 +83,7 @@ func (c Contact) MarshalCompact() ([]byte, error) {
 // NOTE: The compact representation always uses the tcp PeerPort, not the udp Port. This is dumb, but that's how the python daemon does it
 func (c *Contact) UnmarshalCompact(b []byte) error {
 	if len(b) != compactNodeInfoLength {
-		return errors.Wrap(errors.New("invalid compact length"), "")
+		return errors.Err("invalid compact length")
 	}
 	c.IP = net.IPv4(b[0], b[1], b[2], b[3]).To4()
 	c.PeerPort = int(uint16(b[5]) | uint16(b[4])<<8)
@@ -94,8 +93,7 @@ func (c *Contact) UnmarshalCompact(b []byte) error {
 
 // MarshalBencode returns the serialized byte slice representation of a contact.
 func (c Contact) MarshalBencode() ([]byte, error) {
-	b, err := bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
+	return bencode.EncodeBytes([]interface{}{c.ID, c.IP.String(), c.Port})
-	return b, errors.Wrap(err, "")
 }
 
 // UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the contact.
@@ -103,26 +101,26 @@ func (c *Contact) UnmarshalBencode(b []byte) error {
 	var raw []bencode.RawMessage
 	err := bencode.DecodeBytes(b, &raw)
 	if err != nil {
-		return errors.Wrap(err, "")
+		return err
 	}
 
 	if len(raw) != 3 {
-		return errors.Wrap(errors.Newf("contact must have 3 elements; got %d", len(raw)), "")
+		return errors.Err("contact must have 3 elements; got %d", len(raw))
 	}
 
 	err = bencode.DecodeBytes(raw[0], &c.ID)
 	if err != nil {
-		return errors.Wrap(err, "")
+		return err
 	}
 
 	var ipStr string
 	err = bencode.DecodeBytes(raw[1], &ipStr)
 	if err != nil {
-		return errors.Wrap(err, "")
+		return err
 	}
 	c.IP = net.ParseIP(ipStr).To4()
 	if c.IP == nil {
-		return errors.Wrap(errors.New("invalid IP"), "")
+		return errors.Err("invalid IP")
 	}
 
 	return bencode.DecodeBytes(raw[2], &c.Port)

dht/contact_test.go

@@ -5,7 +5,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 func TestCompactEncoding(t *testing.T) {

dht/dht.go

@@ -6,10 +6,10 @@ import (
 	"strings"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 
-	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cast"
 )
@@ -76,7 +76,7 @@ func (dht *DHT) connect(conn UDPConn) error {
 func (dht *DHT) Start() error {
 	listener, err := net.ListenPacket(Network, dht.conf.Address)
 	if err != nil {
-		return errors.WithStack(err)
+		return errors.Err(err)
 	}
 	conn := listener.(*net.UDPConn)
 
@@ -117,7 +117,7 @@ func (dht *DHT) join() {
 	for _, addr := range dht.conf.SeedNodes {
 		err := dht.Ping(addr)
 		if err != nil {
-			log.Error(errors.WithMessage(err, fmt.Sprintf("[%s] join", dht.node.id.HexShort())))
+			log.Error(errors.Prefix(fmt.Sprintf("[%s] join", dht.node.id.HexShort()), err))
 		} else {
 			atLeastOneNodeResponded = true
 		}
@@ -165,7 +165,7 @@ func (dht *DHT) Ping(addr string) error {
 	tmpNode := Contact{ID: bits.Rand(), IP: raddr.IP, Port: raddr.Port}
 	res := dht.node.Send(tmpNode, Request{Method: pingMethod}, SendOptions{skipIDCheck: true})
 	if res == nil {
-		return errors.WithStack(errors.Newf("no response from node %s", addr))
+		return errors.Err("no response from node %s", addr)
 	}
 
 	return nil
@@ -210,21 +210,21 @@ func getContact(nodeID, addr string) (Contact, error) {
 
 	ip, port, err := net.SplitHostPort(addr)
 	if err != nil {
-		return c, errors.WithStack(err)
+		return c, errors.Err(err)
 	} else if ip == "" {
-		return c, errors.WithStack(errors.New("address does not contain an IP"))
+		return c, errors.Err("address does not contain an IP")
 	} else if port == "" {
-		return c, errors.WithStack(errors.New("address does not contain a port"))
+		return c, errors.Err("address does not contain a port")
 	}
 
 	c.IP = net.ParseIP(ip)
 	if c.IP == nil {
-		return c, errors.WithStack(errors.New("invalid ip"))
+		return c, errors.Err("invalid ip")
 	}
 
 	c.Port, err = cast.ToIntE(port)
 	if err != nil {
-		return c, errors.WithStack(err)
+		return c, errors.Err(err)
 	}
 
 	return c, nil

dht/dht_announce.go

@@ -7,11 +7,10 @@ import (
 	"sync"
 	"time"
 
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
 	"golang.org/x/time/rate"
-
-	"github.com/lbryio/lbry.go/v3/dht/bits"
-
-	"github.com/cockroachdb/errors"
 )
 
 type queueEdit struct {
@@ -61,7 +60,7 @@ func (dht *DHT) runAnnouncer() {
 		for {
 			err := limiter.Wait(context.Background()) // TODO: should use grp.ctx somehow? so when grp is closed, wait returns
 			if err != nil {
-				log.Error(errors.WithMessage(err, "rate limiter"))
+				log.Error(errors.Prefix("rate limiter", err))
 				continue
 			}
 			select {
@@ -146,7 +145,7 @@ func (dht *DHT) runAnnouncer() {
 				defer dht.grp.Done()
 				err := dht.announce(hash)
 				if err != nil {
-					log.Error(errors.WithMessage(err, "announce"))
+					log.Error(errors.Prefix("announce", err))
 				}
 
 				if dht.conf.AnnounceNotificationCh != nil {

dht/dht_test.go

@@ -6,7 +6,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 func TestNodeFinder_FindNodes(t *testing.T) {

dht/message.go

@@ -7,9 +7,9 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 
-	"github.com/cockroachdb/errors"
 	"github.com/lyoshenka/bencode"
 	"github.com/spf13/cast"
 )
@@ -62,13 +62,13 @@ func (m *messageID) UnmarshalBencode(encoded []byte) error {
 	var str string
 	err := bencode.DecodeBytes(encoded, &str)
 	if err != nil {
-		return errors.Wrap(err, "")
+		return err
 	}
 	copy(m[:], str)
 	return nil
 }
 
-// MarshalBencode returns the encoded byte slice of the message id.
+// MarshallBencode returns the encoded byte slice of the message id.
 func (m messageID) MarshalBencode() ([]byte, error) {
 	str := string(m[:])
 	return bencode.EncodeBytes(str)
@@ -103,14 +103,13 @@ func (r Request) MarshalBencode() ([]byte, error) {
 	} else {
 		args = []string{} // request must always have keys 0-4, so we use an empty list for PING
 	}
-	b, err := bencode.EncodeBytes(map[string]interface{}{
+	return bencode.EncodeBytes(map[string]interface{}{
 		headerTypeField:      requestType,
 		headerMessageIDField: r.ID,
 		headerNodeIDField:    r.NodeID,
 		headerPayloadField:   r.Method,
 		headerArgsField:      args,
 	})
-	return b, errors.Wrap(err, "bencode")
 }
 
 // UnmarshalBencode unmarshals the serialized byte slice into the appropriate fields of the request.
@@ -123,7 +122,7 @@ func (r *Request) UnmarshalBencode(b []byte) error {
 	}
 	err := bencode.DecodeBytes(b, &raw)
 	if err != nil {
-		return errors.Wrap(err, "request unmarshal")
+		return errors.Prefix("request unmarshal", err)
 	}
 
 	r.ID = raw.ID
@@ -134,12 +133,12 @@ func (r *Request) UnmarshalBencode(b []byte) error {
 		r.StoreArgs = &storeArgs{} // bencode wont find the unmarshaler on a null pointer. need to fix it.
 		err = bencode.DecodeBytes(raw.Args, &r.StoreArgs)
 		if err != nil {
-			return errors.Wrap(err, "request unmarshal")
+			return errors.Prefix("request unmarshal", err)
 		}
 	} else if len(raw.Args) > 2 { // 2 because an empty list is `le`
 		r.Arg, r.ProtocolVersion, err = processArgsAndProtoVersion(raw.Args)
 		if err != nil {
-			return errors.WithMessage(err, "request unmarshal")
+			return errors.Prefix("request unmarshal", err)
 		}
 	}
 
@@ -150,7 +149,7 @@ func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, versi
 	var args []bencode.RawMessage
 	err = bencode.DecodeBytes(raw, &args)
 	if err != nil {
-		return nil, 0, errors.Wrap(err, "")
+		return nil, 0, err
 	}
 
 	if len(args) == 0 {
@@ -170,7 +169,7 @@ func processArgsAndProtoVersion(raw bencode.RawMessage) (arg *bits.Bitmap, versi
 		var b bits.Bitmap
 		err = bencode.DecodeBytes(args[0], &b)
 		if err != nil {
-			return nil, 0, errors.Wrap(err, "")
+			return nil, 0, err
 		}
 		arg = &b
 	}
@@ -225,39 +224,39 @@ func (s *storeArgs) UnmarshalBencode(b []byte) error {
 	var argsInt []bencode.RawMessage
 	err := bencode.DecodeBytes(b, &argsInt)
 	if err != nil {
-		return errors.Wrap(err, "storeArgs unmarshal")
+		return errors.Prefix("storeArgs unmarshal", err)
 	}
 
 	if len(argsInt) != 4 {
-		return errors.Wrap(errors.Newf("unexpected number of fields for store args. got %d", len(argsInt)), "")
+		return errors.Err("unexpected number of fields for store args. got " + cast.ToString(len(argsInt)))
 	}
 
 	err = bencode.DecodeBytes(argsInt[0], &s.BlobHash)
 	if err != nil {
-		return errors.Wrap(err, "storeArgs unmarshal")
+		return errors.Prefix("storeArgs unmarshal", err)
 	}
 
 	err = bencode.DecodeBytes(argsInt[1], &s.Value)
 	if err != nil {
-		return errors.Wrap(err, "storeArgs unmarshal")
+		return errors.Prefix("storeArgs unmarshal", err)
 	}
 
 	err = bencode.DecodeBytes(argsInt[2], &s.NodeID)
 	if err != nil {
-		return errors.Wrap(err, "storeArgs unmarshal")
+		return errors.Prefix("storeArgs unmarshal", err)
 	}
 
 	var selfStore int
 	err = bencode.DecodeBytes(argsInt[3], &selfStore)
 	if err != nil {
-		return errors.Wrap(err, "storeArgs unmarshal")
+		return errors.Prefix("storeArgs unmarshal", err)
 	}
 	if selfStore == 0 {
 		s.SelfStore = false
 	} else if selfStore == 1 {
 		s.SelfStore = true
 	} else {
-		return errors.Wrap(errors.New("selfstore must be 1 or 0"), "")
+		return errors.Err("selfstore must be 1 or 0")
 	}
 
 	return nil
@@ -312,7 +311,7 @@ func (r Response) MarshalBencode() ([]byte, error) {
 	} else if r.FindValueKey != "" {
 		// findValue success
 		if r.Token == "" {
-			return nil, errors.WithStack(errors.New("response to findValue must have a token"))
+			return nil, errors.Err("response to findValue must have a token")
 		}
 
 		var contacts [][]byte
@@ -455,7 +454,7 @@ func (e *Error) UnmarshalBencode(b []byte) error {
 	}
 	err := bencode.DecodeBytes(b, &raw)
 	if err != nil {
-		return errors.Wrap(err, "")
+		return err
 	}
 
 	e.ID = raw.ID

dht/message_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/lyoshenka/bencode"

dht/node.go

@@ -7,10 +7,12 @@ import (
 	"sync"
 	"time"
 
-	"github.com/cockroachdb/errors"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/util"
+
 	"github.com/davecgh/go-spew/spew"
-	"github.com/lbryio/lbry.go/v3/dht/bits"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
 	"github.com/lyoshenka/bencode"
 )
 
@@ -161,8 +163,7 @@ func (n *Node) Shutdown() {
 func (n *Node) handlePacket(pkt packet) {
 	//log.Debugf("[%s] Received message from %s (%d bytes) %s", n.id.HexShort(), pkt.raddr.String(), len(pkt.data), hex.EncodeToString(pkt.data))
 
-	firstFive := string(pkt.data[0:5])
+	if !util.InSlice(string(pkt.data[0:5]), []string{"d1:0i", "di0ei"}) {
-	if firstFive != "d1:0i" && firstFive != "di0ei" {
 		log.Errorf("[%s] data is not a well-formatted dict: (%d bytes) %s", n.id.HexShort(), len(pkt.data), hex.EncodeToString(pkt.data))
 		return
 	}
@@ -317,7 +318,7 @@ func (n *Node) handleError(addr *net.UDPAddr, e Error) {
 func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
 	encoded, err := bencode.EncodeBytes(data)
 	if err != nil {
-		return errors.WithStack(err)
+		return errors.Err(err)
 	}
 
 	if req, ok := data.(Request); ok {
@@ -339,7 +340,7 @@ func (n *Node) sendMessage(addr *net.UDPAddr, data Message) error {
 	}
 
 	_, err = n.conn.WriteToUDP(encoded, addr)
-	return errors.WithStack(err)
+	return errors.Err(err)
 }
 
 // transaction represents a single query to the dht. it stores the queried contact, the request, and the response channel

dht/node_finder.go

@@ -1,15 +1,14 @@
 package dht
 
 import (
-	"crypto/rand"
-	"encoding/hex"
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/crypto"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 
-	"github.com/cockroachdb/errors"
 	"github.com/sirupsen/logrus"
 	"go.uber.org/atomic"
 )
@@ -79,10 +78,7 @@ func (cf *contactFinder) Find() ([]Contact, bool, error) {
 
 	cf.appendNewToShortlist(cf.node.rt.GetClosest(cf.target, alpha))
 	if len(cf.shortlist) == 0 {
-		return nil, false, errors.WithStack(errors.Newf(
+		return nil, false, errors.Err("[%s] find %s: no contacts in routing table", cf.node.id.HexShort(), cf.target.HexShort())
-			"[%s] find %s: no contacts in routing table",
-			cf.node.id.HexShort(), cf.target.HexShort(),
-		))
 	}
 
 	go cf.cycle(false)
@@ -117,7 +113,7 @@ CycleLoop:
 
 // cycle does a single cycle of sending alpha probes and checking results against closestNode
 func (cf *contactFinder) cycle(bigCycle bool) {
-	cycleID := randString(6)
+	cycleID := crypto.RandString(6)
 	if bigCycle {
 		cf.debug("LAUNCHING CYCLE %s, AND ITS A BIG CYCLE", cycleID)
 	} else {
@@ -340,19 +336,3 @@ func (cf *contactFinder) closest(contacts ...Contact) *Contact {
 	}
 	return &closest
 }
-
-// randString returns a random alphanumeric string of a given length
-func randString(length int) string {
-	buf := make([]byte, length/2)
-	_, err := rand.Reader.Read(buf)
-	if err != nil {
-		panic(err)
-	}
-
-	randStr := hex.EncodeToString(buf)[:length]
-	if len(randStr) < length {
-		panic("Could not create random string that is long enough")
-	}
-
-	return randStr
-}

dht/node_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 	"github.com/lyoshenka/bencode"
 )
 

dht/routing_table.go

@@ -9,10 +9,9 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
-
+	"github.com/lbryio/lbry.go/v2/extras/stop"
-	"github.com/cockroachdb/errors"
 )
 
 // TODO: if routing table is ever empty (aka the node is isolated), it should re-bootstrap
@@ -41,7 +40,7 @@ func (p *peer) Touch() {
 	p.NumFailures = 0
 }
 
-// ActiveInLast returns whether a peer has responded in the last `d` duration
+// ActiveSince returns whether a peer has responded in the last `d` duration
 // this is used to check if the peer is "good", meaning that we believe the peer will respond to our requests
 func (p *peer) ActiveInLast(d time.Duration) bool {
 	return time.Since(p.LastActivity) < d
@@ -107,7 +106,7 @@ func (b *bucket) UpdatePeer(p peer, insertIfNew bool) error {
 	defer b.lock.Unlock()
 
 	if !b.Range.Contains(p.Distance) {
-		return errors.WithStack(errors.New("this bucket range does not cover this peer"))
+		return errors.Err("this bucket range does not cover this peer")
 	}
 
 	peerIndex := find(p.Contact.ID, b.peers)
@@ -405,27 +404,27 @@ func (rt *routingTable) UnmarshalJSON(b []byte) error {
 
 	rt.id, err = bits.FromHex(data.ID)
 	if err != nil {
-		return errors.WithMessage(err, "decoding ID")
+		return errors.Prefix("decoding ID", err)
 	}
 	rt.reset()
 
 	for _, s := range data.Contacts {
 		parts := strings.Split(s, rtContactSep)
 		if len(parts) != 3 {
-			return errors.WithStack(errors.Newf("decoding contact %s: wrong number of parts", s))
+			return errors.Err("decoding contact %s: wrong number of parts", s)
 		}
 		var c Contact
 		c.ID, err = bits.FromHex(parts[0])
 		if err != nil {
-			return errors.WithStack(errors.Newf("decoding contact %s: invalid ID: %s", s, err))
+			return errors.Err("decoding contact %s: invalid ID: %s", s, err)
 		}
 		c.IP = net.ParseIP(parts[1])
 		if c.IP == nil {
-			return errors.WithStack(errors.Newf("decoding contact %s: invalid IP", s))
+			return errors.Err("decoding contact %s: invalid IP", s)
 		}
 		c.Port, err = strconv.Atoi(parts[2])
 		if err != nil {
-			return errors.WithStack(errors.Newf("decoding contact %s: invalid port: %s", s, err))
+			return errors.Err("decoding contact %s: invalid port: %s", s, err)
 		}
 		rt.Update(c)
 	}

dht/routing_table_test.go

@@ -8,7 +8,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 
 	"github.com/sebdah/goldie"
 )

dht/rpc.go

@@ -7,9 +7,9 @@ import (
 	"strconv"
 	"sync"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 
-	"github.com/cockroachdb/errors"
 	"github.com/gorilla/mux"
 	rpc2 "github.com/gorilla/rpc/v2"
 	"github.com/gorilla/rpc/v2/json"
@@ -25,7 +25,7 @@ type RpcPingArgs struct {
 
 func (rpc *rpcReceiver) Ping(r *http.Request, args *RpcPingArgs, result *string) error {
 	if args.Address == "" {
-		return errors.WithStack(errors.New("no address given"))
+		return errors.Err("no address given")
 	}
 
 	err := rpc.dht.Ping(args.Address)
@@ -92,7 +92,7 @@ func (rpc *rpcReceiver) FindValue(r *http.Request, args *RpcFindArgs, result *Rp
 		return nil
 	}
 
-	return errors.WithStack(errors.New("not sure what happened"))
+	return errors.Err("not sure what happened")
 }
 
 type RpcIterativeFindValueArgs struct {
@@ -166,7 +166,7 @@ func (dht *DHT) runRPCServer(port int) {
 	s.RegisterCodec(json.NewCodec(), "application/json;charset=UTF-8")
 	err := s.RegisterService(&rpcReceiver{dht: dht}, "rpc")
 	if err != nil {
-		log.Error(errors.WithMessage(err, "registering rpc service"))
+		log.Error(errors.Prefix("registering rpc service", err))
 		return
 	}
 
@@ -188,7 +188,7 @@ func (dht *DHT) runRPCServer(port int) {
 	<-dht.grp.Ch()
 	err = server.Shutdown(context.Background())
 	if err != nil {
-		log.Error(errors.WithMessage(err, "shutting down rpc service"))
+		log.Error(errors.Prefix("shutting down rpc service", err))
 		return
 	}
 	wg.Wait()

dht/store.go

@@ -3,7 +3,7 @@ package dht
 import (
 	"sync"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
 )
 
 // TODO: expire stored data after tExpire time

dht/testing.go

@@ -7,9 +7,8 @@ import (
 	"testing"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
-
+	"github.com/lbryio/lbry.go/v2/extras/errors"
-	"github.com/cockroachdb/errors"
 )
 
 var testingDHTIP = "127.0.0.1"
@@ -122,12 +121,12 @@ func (t testUDPConn) ReadFromUDP(b []byte) (int, *net.UDPAddr, error) {
 	select {
 	case packet, ok := <-t.toRead:
 		if !ok {
-			return 0, nil, errors.WithStack(errors.New("conn closed"))
+			return 0, nil, errors.Err("conn closed")
 		}
 		n := copy(b, packet.data)
 		return n, packet.addr, nil
 	case <-timeoutCh:
-		return 0, nil, timeoutErr{errors.WithStack(errors.New("timeout"))}
+		return 0, nil, timeoutErr{errors.Err("timeout")}
 	}
 }
 

dht/token_cache.go

@@ -4,8 +4,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 )
 
 // TODO: this should be moved out of dht and into node, and it should be completely hidden inside node. dht should not need to know about tokens

dht/token_manager.go

@@ -9,8 +9,8 @@ import (
 	"sync"
 	"time"
 
-	"github.com/lbryio/lbry.go/v3/dht/bits"
+	"github.com/lbryio/lbry.go/v2/dht/bits"
-	"github.com/lbryio/lbry.go/v3/extras/stop"
+	"github.com/lbryio/lbry.go/v2/extras/stop"
 )
 
 type tokenManager struct {

electrum/client.go

@@ -1,149 +0,0 @@
-package electrum
-
-import (
-	"encoding/base64"
-	"encoding/hex"
-
-	"github.com/lbryio/lbry.go/v3/lbrycrd"
-	"github.com/lbryio/lbry.go/v3/schema/stake"
-	types "github.com/lbryio/types/v2/go"
-
-	"github.com/cockroachdb/errors"
-	"github.com/golang/protobuf/proto"
-	"github.com/lbryio/lbcutil"
-	"github.com/spf13/cast"
-)
-
-// Raw makes a raw wallet server request
-func (n *Node) Raw(method string, params []string, v interface{}) error {
-	return n.request(method, params, v)
-}
-
-// ServerVersion returns the server's version.
-// https://electrumx.readthedocs.io/en/latest/protocol-methods.html#server-version
-func (n *Node) ServerVersion() (string, error) {
-	resp := &struct {
-		Result []string `json:"result"`
-	}{}
-	err := n.request("server.version", []string{"reflector.go", ProtocolVersion}, resp)
-
-	var v string
-	if len(resp.Result) >= 2 {
-		v = resp.Result[1]
-	}
-
-	return v, err
-}
-
-func (n *Node) Resolve(url string) (*types.Output, error) {
-	outputs := &types.Outputs{}
-	resp := &struct {
-		Result string `json:"result"`
-	}{}
-
-	err := n.request("blockchain.claimtrie.resolve", []string{url}, resp)
-	if err != nil {
-		return nil, err
-	}
-
-	b, err := base64.StdEncoding.DecodeString(resp.Result)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	err = proto.Unmarshal(b, outputs)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	if len(outputs.GetTxos()) != 1 {
-		return nil, errors.New("expected 1 output, got " + cast.ToString(len(outputs.GetTxos())))
-	}
-
-	if e := outputs.GetTxos()[0].GetError(); e != nil {
-		return nil, errors.Newf("%s: %s", e.GetCode(), e.GetText())
-	}
-
-	return outputs.GetTxos()[0], nil
-}
-
-type GetClaimsInTxResp struct {
-	Jsonrpc string `json:"jsonrpc"`
-	ID      int    `json:"id"`
-	Result  []struct {
-		Name            string        `json:"name"`
-		ClaimID         string        `json:"claim_id"`
-		Txid            string        `json:"txid"`
-		Nout            int           `json:"nout"`
-		Amount          int           `json:"amount"`
-		Depth           int           `json:"depth"`
-		Height          int           `json:"height"`
-		Value           string        `json:"value"`
-		ClaimSequence   int           `json:"claim_sequence"`
-		Address         string        `json:"address"`
-		Supports        []interface{} `json:"supports"` // TODO: finish me
-		EffectiveAmount int           `json:"effective_amount"`
-		ValidAtHeight   int           `json:"valid_at_height"`
-	} `json:"result"`
-}
-
-func (n *Node) GetClaimsInTx(txid string) (*GetClaimsInTxResp, error) {
-	var resp GetClaimsInTxResp
-	err := n.request("blockchain.claimtrie.getclaimsintx", []string{txid}, &resp)
-	return &resp, err
-}
-
-func (n *Node) GetTx(txid string) (string, error) {
-	resp := &struct {
-		Result string `json:"result"`
-	}{}
-
-	err := n.request("blockchain.transaction.get", []string{txid}, resp)
-	if err != nil {
-		return "", err
-	}
-
-	return resp.Result, nil
-}
-
-func (n *Node) GetClaimInTx(txid string, nout int) (*types.Claim, error) {
-	hexTx, err := n.GetTx(txid)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	rawTx, err := hex.DecodeString(hexTx)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	tx, err := lbcutil.NewTxFromBytes(rawTx)
-	if err != nil {
-		return nil, errors.WithStack(err)
-	}
-
-	if len(tx.MsgTx().TxOut) <= nout {
-		return nil, errors.WithStack(errors.New("nout not found"))
-	}
-
-	script := tx.MsgTx().TxOut[nout].PkScript
-
-	var value []byte
-	if lbrycrd.IsClaimNameScript(script) {
-		_, value, _, err = lbrycrd.ParseClaimNameScript(script)
-	} else if lbrycrd.IsClaimUpdateScript(script) {
-		_, _, value, _, err = lbrycrd.ParseClaimUpdateScript(script)
-	} else {
-		err = errors.New("no claim found in output")
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	ch, err := stake.DecodeClaimBytes(value, "")
-	if err != nil {
-		return nil, err
-	}
-
-	return ch.Claim, nil
-}

electrum/network.go

@@ -1,279 +0,0 @@
-package electrum
-
-// copied from https://github.com/d4l3k/go-electrum
-
-import (
-	"crypto/tls"
-	"encoding/json"
-	"math/rand"
-	"net"
-	"sync"
-	"time"
-
-	"github.com/lbryio/lbry.go/v3/extras/stop"
-
-	"github.com/cockroachdb/errors"
-	log "github.com/sirupsen/logrus"
-	"go.uber.org/atomic"
-)
-
-const (
-	ClientVersion   = "0.0.1"
-	ProtocolVersion = "1.0"
-)
-
-var (
-	ErrNotImplemented = errors.New("not implemented")
-	ErrNodeConnected  = errors.New("node already connected")
-	ErrConnectFailed  = errors.New("failed to connect")
-	ErrTimeout        = errors.New("timeout")
-)
-
-type response struct {
-	data []byte
-	err  error
-}
-
-type Node struct {
-	transport *TCPTransport
-	nextId    atomic.Uint32
-	grp       *stop.Group
-
-	handlersMu *sync.RWMutex
-	handlers   map[uint32]chan response
-
-	pushHandlersMu *sync.RWMutex
-	pushHandlers   map[string][]chan response
-
-	timeout time.Duration
-}
-
-// NewNode creates a new node.
-func NewNode() *Node {
-	return &Node{
-		handlers:       make(map[uint32]chan response),
-		pushHandlers:   make(map[string][]chan response),
-		handlersMu:     &sync.RWMutex{},
-		pushHandlersMu: &sync.RWMutex{},
-		grp:            stop.New(),
-		timeout:        1 * time.Second,
-	}
-}
-
-// Connect creates a new connection to the specified address.
-func (n *Node) Connect(addrs []string, config *tls.Config) error {
-	if n.transport != nil {
-		return errors.WithStack(ErrNodeConnected)
-	}
-
-	// shuffle addresses for load balancing
-	rand.Shuffle(len(addrs), func(i, j int) { addrs[i], addrs[j] = addrs[j], addrs[i] })
-
-	var err error
-
-	for _, addr := range addrs {
-		n.transport, err = NewTransport(addr, config)
-		if err == nil {
-			break
-		}
-		if errors.Is(err, ErrTimeout) {
-			continue
-		}
-		if e, ok := err.(*net.OpError); ok && e.Err.Error() == "no such host" {
-			// net.errNoSuchHost is not exported, so we have to string-match
-			continue
-		}
-		return errors.WithStack(err)
-	}
-
-	if n.transport == nil {
-		return errors.WithStack(ErrConnectFailed)
-	}
-
-	log.Debugf("wallet connected to %s", n.transport.conn.RemoteAddr())
-
-	n.grp.Add(1)
-	go func() {
-		defer n.grp.Done()
-		<-n.grp.Ch()
-		n.transport.Shutdown()
-	}()
-
-	n.grp.Add(1)
-	go func() {
-		defer n.grp.Done()
-		n.handleErrors()
-	}()
-
-	n.grp.Add(1)
-	go func() {
-		defer n.grp.Done()
-		n.listen()
-	}()
-
-	return nil
-}
-
-func (n *Node) Shutdown() {
-	var addr net.Addr
-	if n.transport != nil {
-		addr = n.transport.conn.RemoteAddr()
-	}
-	log.Debugf("shutting down wallet %s", addr)
-	n.grp.StopAndWait()
-	log.Debugf("wallet stopped")
-}
-
-func (n *Node) handleErrors() {
-	for {
-		select {
-		case <-n.grp.Ch():
-			return
-		case err := <-n.transport.Errors():
-			n.err(errors.WithStack(err))
-		}
-	}
-}
-
-// err handles errors produced by the foreign node.
-func (n *Node) err(err error) {
-	// TODO: Better error handling.
-	log.Error(errors.WithStack(err))
-}
-
-// listen processes messages from the server.
-func (n *Node) listen() {
-	for {
-		select {
-		case <-n.grp.Ch():
-			return
-		default:
-		}
-
-		select {
-		case <-n.grp.Ch():
-			return
-		case bytes := <-n.transport.Responses():
-			msg := &struct {
-				Id     uint32 `json:"id"`
-				Method string `json:"method"`
-				Error  struct {
-					Code    int    `json:"code"`
-					Message string `json:"message"`
-				} `json:"error"`
-			}{}
-			msg2 := &struct {
-				Id     uint32 `json:"id"`
-				Method string `json:"method"`
-				Error  struct {
-					Code    int `json:"code"`
-					Message struct {
-						Code    int    `json:"code"`
-						Message string `json:"message"`
-					} `json:"message"`
-				} `json:"error"`
-			}{}
-			r := response{}
-
-			err := json.Unmarshal(bytes, msg)
-			if err != nil {
-				// try msg2, a hack around the weird error-in-error response we sometimes get from wallet server
-				// maybe that happens because the wallet server passes a lbrycrd error through to us?
-				if err2 := json.Unmarshal(bytes, msg2); err2 == nil {
-					err = nil
-					msg.Id = msg2.Id
-					msg.Method = msg2.Method
-					msg.Error = msg2.Error.Message
-				}
-			}
-
-			if err != nil {
-				r.err = errors.WithStack(err)
-				n.err(r.err)
-			} else if len(msg.Error.Message) > 0 {
-				r.err = errors.WithStack(errors.Newf("%d: %s", msg.Error.Code, msg.Error.Message))
-			} else {
-				r.data = bytes
-			}
-
-			if len(msg.Method) > 0 {
-				n.pushHandlersMu.RLock()
-				handlers := n.pushHandlers[msg.Method]
-				n.pushHandlersMu.RUnlock()
-
-				for _, handler := range handlers {
-					select {
-					case handler <- r:
-					default:
-					}
-				}
-			}
-
-			n.handlersMu.RLock()
-			c, ok := n.handlers[msg.Id]
-			n.handlersMu.RUnlock()
-			if ok {
-				c <- r
-			}
-		}
-	}
-}
-
-// listenPush returns a channel of messages matching the method.
-//func (n *Node) listenPush(method string) <-chan []byte {
-//	c := make(chan []byte, 1)
-//	n.pushHandlersMu.Lock()
-//	defer n.pushHandlersMu.Unlock()
-//	n.pushHandlers[method] = append(n.pushHandlers[method], c)
-//	return c
-//}
-
-// request makes a request to the server and unmarshals the response into v.
-func (n *Node) request(method string, params []string, v interface{}) error {
-	msg := struct {
-		Id     uint32   `json:"id"`
-		Method string   `json:"method"`
-		Params []string `json:"params"`
-	}{
-		Id:     n.nextId.Load(),
-		Method: method,
-		Params: params,
-	}
-	n.nextId.Inc()
-
-	bytes, err := json.Marshal(msg)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	bytes = append(bytes, delimiter)
-
-	c := make(chan response, 1)
-
-	n.handlersMu.Lock()
-	n.handlers[msg.Id] = c
-	n.handlersMu.Unlock()
-
-	err = n.transport.Send(bytes)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	var r response
-	select {
-	case <-n.grp.Ch():
-		return nil
-	case r = <-c:
-	case <-time.After(n.timeout):
-		r = response{err: ErrTimeout}
-	}
-
-	n.handlersMu.Lock()
-	delete(n.handlers, msg.Id)
-	n.handlersMu.Unlock()
-
-	if r.err != nil {
-		return errors.WithStack(r.err)
-	}
-
-	return errors.WithStack(json.Unmarshal(r.data, v))
-}

electrum/transport.go

@@ -1,136 +0,0 @@
-package electrum
-
-// copied from https://github.com/d4l3k/go-electrum
-
-import (
-	"bufio"
-	"crypto/tls"
-	"encoding/json"
-	"net"
-	"time"
-
-	"github.com/lbryio/lbry.go/v3/extras/stop"
-
-	"github.com/cockroachdb/errors"
-	log "github.com/sirupsen/logrus"
-)
-
-type TCPTransport struct {
-	conn      net.Conn
-	responses chan []byte
-	errors    chan error
-	grp       *stop.Group
-}
-
-func NewTransport(addr string, config *tls.Config) (*TCPTransport, error) {
-	var conn net.Conn
-	var err error
-
-	timeout := 5 * time.Second
-	if config != nil {
-		conn, err = tls.DialWithDialer(&net.Dialer{Timeout: timeout}, "tcp", addr, config)
-	} else {
-		conn, err = net.DialTimeout("tcp", addr, timeout)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	t := &TCPTransport{
-		conn:      conn,
-		responses: make(chan []byte),
-		errors:    make(chan error),
-		grp:       stop.New(),
-	}
-
-	t.grp.Add(1)
-	go func() {
-		defer t.grp.Done()
-		<-t.grp.Ch()
-		t.close()
-	}()
-
-	t.grp.Add(1)
-	go func() {
-		defer t.grp.Done()
-		t.listen()
-	}()
-
-	err = t.test()
-	if err != nil {
-		t.grp.StopAndWait()
-		return nil, errors.WithMessage(err, addr)
-	}
-
-	return t, nil
-}
-
-const delimiter = byte('\n')
-
-func (t *TCPTransport) Send(body []byte) error {
-	log.Debugf("%s <- %s", t.conn.RemoteAddr(), body)
-	_, err := t.conn.Write(body)
-	return err
-}
-
-func (t *TCPTransport) Responses() <-chan []byte { return t.responses }
-func (t *TCPTransport) Errors() <-chan error     { return t.errors }
-func (t *TCPTransport) Shutdown()                { t.grp.StopAndWait() }
-
-func (t *TCPTransport) listen() {
-	reader := bufio.NewReader(t.conn)
-	for {
-		line, err := reader.ReadBytes(delimiter)
-		if err != nil {
-			t.error(err)
-			return
-		}
-
-		log.Debugf("%s -> %s", t.conn.RemoteAddr(), line)
-
-		t.responses <- line
-	}
-}
-
-func (t *TCPTransport) error(err error) {
-	select {
-	case t.errors <- err:
-	default:
-	}
-}
-
-func (t *TCPTransport) test() error {
-	err := t.Send([]byte(`{"id":1,"method":"server.version"}` + "\n"))
-	if err != nil {
-		return errors.WithStack(err)
-	}
-
-	var data []byte
-	select {
-	case data = <-t.Responses():
-	case <-time.Tick(1 * time.Second):
-		return errors.WithStack(ErrTimeout)
-	}
-
-	var response struct {
-		Error struct {
-			Message string `json:"message"`
-		} `json:"error"`
-	}
-
-	err = json.Unmarshal(data, &response)
-	if err != nil {
-		return errors.WithStack(err)
-	}
-	if response.Error.Message != "" {
-		return errors.WithStack(errors.New(response.Error.Message))
-	}
-	return nil
-}
-
-func (t *TCPTransport) close() {
-	err := t.conn.Close()
-	if err != nil {
-		t.error(err)
-	}
-}

extras/api/server.go

@@ -0,0 +1,337 @@
+package api
+
+import (
+	"encoding/json"
+	"net/http"
+	"reflect"
+	"strings"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/util"
+	"github.com/lbryio/lbry.go/v2/extras/validator"
+	v "github.com/lbryio/ozzo-validation"
+
+	"github.com/spf13/cast"
+)
+
+// ResponseHeaders are returned with each response
+var ResponseHeaders map[string]string
+
+// CorsDomains Allowed domains for CORS Policy
+var CorsDomains []string
+
+// CorsAllowLocalhost if true localhost connections are always allowed
+var CorsAllowLocalhost bool
+
+// Log allows logging of events and errors
+var Log = func(*http.Request, *Response, error) {}
+
+// http://choly.ca/post/go-json-marshalling/
+type ResponseInfo struct {
+	Success bool        `json:"success"`
+	Error   *string     `json:"error"`
+	Data    interface{} `json:"data"`
+	Trace   []string    `json:"_trace,omitempty"`
+}
+
+// BuildJSONResponse allows implementers to control the json response form from the api
+var BuildJSONResponse = func(response ResponseInfo) ([]byte, error) {
+	return json.MarshalIndent(&response, "", "  ")
+}
+
+// TraceEnabled Attaches a trace field to the JSON response when enabled.
+var TraceEnabled = false
+
+// StatusError represents an error with an associated HTTP status code.
+type StatusError struct {
+	Status int
+	Err    error
+}
+
+func (se StatusError) Error() string { return se.Err.Error() }
+
+// Response is returned by API handlers
+type Response struct {
+	Status      int
+	Data        interface{}
+	RedirectURL string
+	Error       error
+}
+
+// Handler handles API requests
+type Handler func(r *http.Request) Response
+
+func (h Handler) callHandlerSafely(r *http.Request) (rsp Response) {
+	defer func() {
+		if r := recover(); r != nil {
+			err, ok := r.(error)
+			if !ok {
+				err = errors.Err("%v", r)
+			}
+			rsp = Response{Error: errors.Wrap(err, 2)}
+		}
+	}()
+
+	return h(r)
+}
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	// Set header settings
+	if ResponseHeaders != nil {
+		//Multiple readers, no writers is okay
+		for key, value := range ResponseHeaders {
+			w.Header().Set(key, value)
+		}
+	}
+	origin := r.Header.Get("origin")
+	for _, d := range CorsDomains {
+		if d == origin {
+			w.Header().Set("Access-Control-Allow-Origin", d)
+			vary := w.Header().Get("Vary")
+			if vary != "*" {
+				if vary != "" {
+					vary += ", "
+				}
+				vary += "Origin"
+			}
+			w.Header().Set("Vary", vary)
+		}
+	}
+
+	if CorsAllowLocalhost && strings.HasPrefix(origin, "http://localhost:") {
+		w.Header().Set("Access-Control-Allow-Origin", origin)
+		vary := w.Header().Get("Vary")
+		if vary != "*" {
+			if vary != "" {
+				vary += ", "
+			}
+			vary += "Origin"
+		}
+		w.Header().Set("Vary", vary)
+	}
+
+	// Stop here if its a preflighted OPTIONS request
+	if r.Method == "OPTIONS" {
+		return
+	}
+
+	rsp := h.callHandlerSafely(r)
+
+	if rsp.Status == 0 {
+		if rsp.Error != nil {
+			ogErr := errors.Unwrap(rsp.Error)
+			if statusError, ok := ogErr.(StatusError); ok {
+				if statusError.Status == 0 {
+					statusError.Status = http.StatusInternalServerError
+				}
+				rsp.Status = statusError.Status
+			} else {
+				rsp.Status = http.StatusInternalServerError
+			}
+		} else if rsp.RedirectURL != "" {
+			rsp.Status = http.StatusFound
+		} else {
+			rsp.Status = http.StatusOK
+		}
+	}
+
+	success := rsp.Status < http.StatusBadRequest
+	if success {
+		Log(r, &rsp, nil)
+	} else {
+		Log(r, &rsp, rsp.Error)
+	}
+
+	// redirect
+	if rsp.Status >= http.StatusMultipleChoices && rsp.Status < http.StatusBadRequest {
+		http.Redirect(w, r, rsp.RedirectURL, rsp.Status)
+		return
+	} else if rsp.RedirectURL != "" {
+		Log(r, &rsp, errors.Base(
+			"status code %d does not indicate a redirect, but RedirectURL is non-empty '%s'",
+			rsp.Status, rsp.RedirectURL,
+		))
+	}
+
+	var errorString *string
+	if rsp.Error != nil {
+		errorStringRaw := rsp.Error.Error()
+		errorString = &errorStringRaw
+	}
+
+	var trace []string
+	if TraceEnabled && errors.HasTrace(rsp.Error) {
+		trace = getTraceFromError(rsp.Error)
+	}
+
+	jsonResponse, err := BuildJSONResponse(ResponseInfo{
+		Success: success,
+		Error:   errorString,
+		Data:    rsp.Data,
+		Trace:   trace,
+	})
+	if err != nil {
+		Log(r, &rsp, errors.Prefix("Error encoding JSON response: ", err))
+		jsonResponse, err = BuildJSONResponse(ResponseInfo{
+			Success: false,
+			Error:   util.PtrToString(err.Error()),
+			Data:    nil,
+			Trace:   getTraceFromError(err),
+		})
+		if err != nil {
+			Log(r, &rsp, errors.Prefix("Error encoding JSON response: ", err))
+		}
+	}
+
+	w.WriteHeader(rsp.Status)
+	_, _ = w.Write(jsonResponse)
+}
+
+func getTraceFromError(err error) []string {
+	trace := strings.Split(errors.Trace(err), "\n")
+	for index, element := range trace {
+		if strings.HasPrefix(element, "\t") {
+			trace[index] = strings.Replace(element, "\t", "    ", 1)
+		}
+	}
+	return trace
+}
+
+// IgnoredFormFields are ignored by FormValues() when checking for extraneous fields
+var IgnoredFormFields []string
+
+func FormValues(r *http.Request, params interface{}, validationRules []*v.FieldRules) error {
+	ref := reflect.ValueOf(params)
+	if !ref.IsValid() || ref.Kind() != reflect.Ptr || ref.Elem().Kind() != reflect.Struct {
+		return errors.Err("'params' must be a pointer to a struct")
+	}
+
+	structType := ref.Elem().Type()
+	structValue := ref.Elem()
+	fields := map[string]bool{}
+	for i := 0; i < structType.NumField(); i++ {
+		fieldName := structType.Field(i).Name
+		formattedName := util.Underscore(fieldName)
+		jsonName, ok := structType.Field(i).Tag.Lookup("json")
+		if ok {
+			formattedName = jsonName
+		}
+		value := strings.TrimSpace(r.FormValue(formattedName))
+
+		// if param is not set at all, continue
+		// comes after call to r.FormValue so form values get parsed internally (if they arent already)
+		if len(r.Form[formattedName]) == 0 {
+			continue
+		}
+
+		fields[formattedName] = true
+		isPtr := false
+		var finalValue reflect.Value
+
+		structField := structValue.FieldByName(fieldName)
+		structFieldKind := structField.Kind()
+		if structFieldKind == reflect.Ptr {
+			isPtr = true
+			structFieldKind = structField.Type().Elem().Kind()
+		}
+
+		switch structFieldKind {
+		case reflect.String:
+			finalValue = reflect.ValueOf(value)
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			if value == "" {
+				continue
+			}
+			castVal, err := cast.ToInt64E(value)
+			if err != nil {
+				return errors.Err("%s: must be an integer", formattedName)
+			}
+			switch structFieldKind {
+			case reflect.Int:
+				finalValue = reflect.ValueOf(int(castVal))
+			case reflect.Int8:
+				finalValue = reflect.ValueOf(int8(castVal))
+			case reflect.Int16:
+				finalValue = reflect.ValueOf(int16(castVal))
+			case reflect.Int32:
+				finalValue = reflect.ValueOf(int32(castVal))
+			case reflect.Int64:
+				finalValue = reflect.ValueOf(castVal)
+			}
+		case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+			if value == "" {
+				continue
+			}
+			castVal, err := cast.ToUint64E(value)
+			if err != nil {
+				return errors.Err("%s: must be an unsigned integer", formattedName)
+			}
+			switch structFieldKind {
+			case reflect.Uint:
+				finalValue = reflect.ValueOf(uint(castVal))
+			case reflect.Uint8:
+				finalValue = reflect.ValueOf(uint8(castVal))
+			case reflect.Uint16:
+				finalValue = reflect.ValueOf(uint16(castVal))
+			case reflect.Uint32:
+				finalValue = reflect.ValueOf(uint32(castVal))
+			case reflect.Uint64:
+				finalValue = reflect.ValueOf(castVal)
+			}
+		case reflect.Bool:
+			if value == "" {
+				continue
+			}
+			if !validator.IsBoolString(value) {
+				return errors.Err("%s: must be one of the following values: %s",
+					formattedName, strings.Join(validator.GetBoolStringValues(), ", "))
+			}
+			finalValue = reflect.ValueOf(validator.IsTruthy(value))
+
+		case reflect.Float32, reflect.Float64:
+			if value == "" {
+				continue
+			}
+			castVal, err := cast.ToFloat64E(value)
+			if err != nil {
+				return errors.Err("%s: must be a floating point number", formattedName)
+			}
+			switch structFieldKind {
+			case reflect.Float32:
+				finalValue = reflect.ValueOf(float32(castVal))
+			case reflect.Float64:
+				finalValue = reflect.ValueOf(float64(castVal))
+			}
+		default:
+			return errors.Err("field %s is an unsupported type", fieldName)
+		}
+
+		if isPtr {
+			if structField.IsNil() {
+				structField.Set(reflect.New(structField.Type().Elem()))
+			}
+			structField.Elem().Set(finalValue)
+		} else {
+			structField.Set(finalValue)
+		}
+	}
+
+	var extraParams []string
+	for k := range r.Form {
+		if _, ok := fields[k]; !ok && !util.InSlice(k, IgnoredFormFields) {
+			extraParams = append(extraParams, k)
+		}
+	}
+	if len(extraParams) > 0 {
+		return errors.Err("Extraneous params: " + strings.Join(extraParams, ", "))
+	}
+
+	if len(validationRules) > 0 {
+		validationErr := v.ValidateStruct(params, validationRules...)
+		if validationErr != nil {
+			return errors.Err(validationErr)
+		}
+	}
+
+	return nil
+}

extras/crypto/crypto.go

@@ -0,0 +1,50 @@
+package crypto
+
+import (
+	"crypto/rand"
+	"encoding/hex"
+	"math/big"
+	"sort"
+	"strings"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"github.com/btcsuite/btcutil/base58"
+	"golang.org/x/crypto/sha3"
+)
+
+// RandString returns a random alphanumeric string of a given length
+func RandString(length int) string {
+	buf := make([]byte, length)
+	_, err := rand.Reader.Read(buf)
+	if err != nil {
+		panic(errors.Err(err))
+	}
+
+	randStr := base58.Encode(buf)[:length]
+	if len(randStr) < length {
+		panic(errors.Err("Could not create random string that is long enough"))
+	}
+
+	return randStr
+}
+
+// Int returns a uniform random value in [0, max). It panics if max <= 0.
+func RandInt64(max int64) int64 {
+	n, err := rand.Int(rand.Reader, big.NewInt(max))
+	if err != nil {
+		panic(err)
+	}
+	return n.Int64()
+}
+
+// HashStringSlice returns a hex hash of a slice of strings
+func HashStringSlice(data []string) string {
+	return hex.EncodeToString(hashStringSliceRaw(data))
+}
+
+func hashStringSliceRaw(data []string) []byte {
+	sort.Strings(data)
+	hash := sha3.Sum256([]byte(strings.Join(data, "")))
+	return hash[:]
+}

extras/errors/README.md

@@ -0,0 +1,4 @@
+# errors
+
+Better error handling. Marries [go-errors/errors](https://github.com/go-errors/errors) to [pkg/errors](https://github.com/pkg/errors), and 
+adds a little bit of our own magic sauce.

extras/errors/errors.go

@@ -0,0 +1,110 @@
+package errors
+
+import (
+	"fmt"
+
+	"github.com/go-errors/errors"
+)
+
+// interop with pkg/errors
+type causer interface {
+	Cause() error
+}
+
+// Err intelligently creates/handles errors, while preserving the stack trace.
+// It works with errors from github.com/pkg/errors too.
+func Err(err interface{}, fmtParams ...interface{}) error {
+	if err == nil {
+		return nil
+	}
+
+	if _, ok := err.(causer); ok {
+		err = fmt.Errorf("%+v", err)
+	} else if errString, ok := err.(string); ok && len(fmtParams) > 0 {
+		err = fmt.Errorf(errString, fmtParams...)
+	}
+
+	return errors.Wrap(err, 1)
+}
+
+// Wrap calls errors.Wrap, in case you want to skip a different amount
+func Wrap(err interface{}, skip int) *errors.Error {
+	if err == nil {
+		return nil
+	}
+
+	if _, ok := err.(causer); ok {
+		err = fmt.Errorf("%+v", err)
+	}
+
+	return errors.Wrap(err, skip+1)
+}
+
+// Unwrap returns the original error that was wrapped
+func Unwrap(err error) error {
+	if err == nil {
+		return nil
+	}
+
+	deeper := true
+	for deeper {
+		deeper = false
+		if e, ok := err.(*errors.Error); ok {
+			err = e.Err
+			deeper = true
+		}
+		if c, ok := err.(causer); ok {
+			err = c.Cause()
+			deeper = true
+		}
+	}
+
+	return err
+}
+
+// Is compares two wrapped errors to determine if the underlying errors are the same
+// It also interops with errors from pkg/errors
+func Is(e error, original error) bool {
+	if c, ok := e.(causer); ok {
+		e = c.Cause()
+	}
+	if c, ok := original.(causer); ok {
+		original = c.Cause()
+	}
+	return errors.Is(e, original)
+}
+
+// Prefix prefixes the message of the error with the given string
+func Prefix(prefix string, err interface{}) error {
+	if err == nil {
+		return nil
+	}
+	return errors.WrapPrefix(Err(err), prefix, 0)
+}
+
+// Trace returns the stack trace
+func Trace(err error) string {
+	if err == nil {
+		return ""
+	}
+	return string(Err(err).(*errors.Error).Stack())
+}
+
+// FullTrace returns the error type, message, and stack trace
+func FullTrace(err error) string {
+	if err == nil {
+		return ""
+	}
+	return Err(err).(*errors.Error).ErrorStack()
+}
+
+// Base returns a simple error with no stack trace attached
+func Base(format string, a ...interface{}) error {
+	return fmt.Errorf(format, a...)
+}
+
+// HasTrace checks if error has a trace attached
+func HasTrace(err error) bool {
+	_, ok := err.(*errors.Error)
+	return ok
+}

extras/jsonrpc/daemon.go

@@ -0,0 +1,779 @@
+package jsonrpc
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"reflect"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fatih/structs"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/mitchellh/mapstructure"
+	"github.com/shopspring/decimal"
+	log "github.com/sirupsen/logrus"
+	"github.com/ybbus/jsonrpc/v2"
+)
+
+const DefaultPort = 5279
+
+const (
+	ErrorWalletNotLoaded     = "WalletNotLoadedError"
+	ErrorWalletAlreadyLoaded = "WalletAlreadyLoadedError"
+	ErrorWalletNotFound      = "WalletNotFoundError"
+	ErrorWalletAlreadyExists = "WalletAlreadyExistsError"
+)
+
+type Client struct {
+	conn    jsonrpc.RPCClient
+	address string
+}
+
+type Error struct {
+	Code    int
+	Name    string
+	Message string
+}
+
+func NewClient(address string) *Client {
+	d := Client{}
+
+	if address == "" {
+		address = "http://localhost:" + strconv.Itoa(DefaultPort)
+	}
+
+	d.conn = jsonrpc.NewClient(address)
+	d.address = address
+
+	return &d
+}
+
+func NewClientAndWait(address string) *Client {
+	d := NewClient(address)
+	for {
+		_, err := d.AccountBalance(nil)
+		if err == nil {
+			return d
+		}
+		time.Sleep(5 * time.Second)
+	}
+}
+
+func Decode(data interface{}, targetStruct interface{}) error {
+	config := &mapstructure.DecoderConfig{
+		Metadata: nil,
+		Result:   targetStruct,
+		TagName:  "json",
+		//WeaklyTypedInput: true,
+		DecodeHook: fixDecodeProto,
+	}
+
+	decoder, err := mapstructure.NewDecoder(config)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+
+	err = decoder.Decode(data)
+	if err != nil {
+		return errors.Wrap(err, 0)
+	}
+	return nil
+}
+
+// WrapError adds error metadata from JSONRPC error response for clients to access
+func WrapError(rpcError *jsonrpc.RPCError) Error {
+	e := Error{Code: rpcError.Code, Message: rpcError.Message}
+	if d, ok := rpcError.Data.(map[string]interface{}); ok {
+		e.Name = d["name"].(string)
+	}
+	return e
+}
+
+func decodeNumber(data interface{}) (decimal.Decimal, error) {
+	var number string
+
+	switch d := data.(type) {
+	case json.Number:
+		number = d.String()
+	case string:
+		number = d
+	default:
+		return decimal.Decimal{}, errors.Err("unexpected number type")
+	}
+
+	dec, err := decimal.NewFromString(number)
+	if err != nil {
+		return decimal.Decimal{}, errors.Wrap(err, 0)
+	}
+
+	return dec, nil
+}
+
+func debugParams(params map[string]interface{}) string {
+	var s []string
+	for k, v := range params {
+		r := reflect.ValueOf(v)
+		if r.Kind() == reflect.Ptr {
+			if r.IsNil() {
+				continue
+			}
+			v = r.Elem().Interface()
+		}
+		s = append(s, fmt.Sprintf("%s=%+v", k, v))
+	}
+	sort.Strings(s)
+	return strings.Join(s, " ")
+}
+
+func (e Error) Error() string {
+	return fmt.Sprintf("Error in daemon: %s", e.Message)
+}
+
+func (d *Client) callNoDecode(command string, params map[string]interface{}) (interface{}, error) {
+	log.Debugln("jsonrpc: " + command + " " + debugParams(params))
+	r, err := d.conn.Call(command, params)
+	if err != nil {
+		return nil, errors.Wrap(err, 0)
+	}
+
+	if r.Error != nil {
+		return nil, WrapError(r.Error)
+	}
+
+	return r.Result, nil
+}
+
+func (d *Client) call(response interface{}, command string, params map[string]interface{}) error {
+	result, err := d.callNoDecode(command, params)
+	if err != nil {
+		return err
+	}
+	return Decode(result, response)
+}
+
+func (d *Client) SetRPCTimeout(timeout time.Duration) {
+	d.conn = jsonrpc.NewClientWithOpts(d.address, &jsonrpc.RPCClientOpts{
+		HTTPClient: &http.Client{Timeout: timeout},
+	})
+}
+
+//============================================
+//				NEW SDK
+//============================================
+
+func (d *Client) AccountSend(accountID *string, amount, toAddress string) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		AccountID *string `json:"account_id"`
+		Amount    string  `json:"amount"`
+		Addresses string  `json:"addresses"`
+	}{
+		AccountID: accountID,
+		Amount:    amount,
+		Addresses: toAddress,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "account_send", structs.Map(args))
+}
+
+func (d *Client) AccountList(page uint64, pageSize uint64) (*AccountListResponse, error) {
+	response := new(AccountListResponse)
+	return response, d.call(response, "account_list", map[string]interface{}{
+		"page":      page,
+		"page_size": pageSize,
+	})
+}
+
+func (d *Client) AccountListForWallet(walletID string) (*AccountListResponse, error) {
+	response := new(AccountListResponse)
+	return response, d.call(response, "account_list", map[string]interface{}{"wallet_id": walletID})
+}
+
+func (d *Client) SingleAccountList(accountID string) (*AccountListResponse, error) {
+	response := new(AccountListResponse)
+	return response, d.call(response, "account_list", map[string]interface{}{"account_id": accountID})
+}
+
+type AccountSettings struct {
+	Default          *bool   `json:"default,omitempty"`
+	NewName          *string `json:"new_name,omitempty"`
+	ReceivingGap     *int    `json:"receiving_gap,omitempty"`
+	ReceivingMaxUses *int    `json:"receiving_max_uses,omitempty"`
+	ChangeGap        *int    `json:"change_gap,omitempty"`
+	ChangeMaxUses    *int    `json:"change_max_uses,omitempty"`
+}
+
+func (d *Client) AccountSet(accountID string, settings AccountSettings) (*Account, error) {
+	response := new(Account)
+	args := struct {
+		AccountID       string `json:"account_id"`
+		AccountSettings `json:",flatten"`
+	}{
+		AccountID:       accountID,
+		AccountSettings: settings,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "account_set", structs.Map(args))
+}
+
+func (d *Client) AccountBalance(account *string) (*AccountBalanceResponse, error) {
+	response := new(AccountBalanceResponse)
+	return response, d.call(response, "account_balance", map[string]interface{}{
+		"account_id": account,
+	})
+}
+
+// funds an account. If everything is true then amount is ignored
+func (d *Client) AccountFund(fromAccount string, toAccount string, amount string, outputs uint64, everything bool) (*AccountFundResponse, error) {
+	response := new(AccountFundResponse)
+	return response, d.call(response, "account_fund", map[string]interface{}{
+		"from_account": fromAccount,
+		"to_account":   toAccount,
+		"amount":       amount,
+		"outputs":      outputs,
+		"everything":   everything,
+		"broadcast":    true,
+	})
+}
+
+func (d *Client) AccountCreate(accountName string, singleKey bool) (*Account, error) {
+	response := new(Account)
+	return response, d.call(response, "account_create", map[string]interface{}{
+		"account_name": accountName,
+		"single_key":   singleKey,
+	})
+}
+
+func (d *Client) AccountRemove(accountID string) (*Account, error) {
+	response := new(Account)
+	return response, d.call(response, "account_remove", map[string]interface{}{
+		"account_id": accountID,
+	})
+}
+
+func (d *Client) AddressUnused(account *string) (*AddressUnusedResponse, error) {
+	response := new(AddressUnusedResponse)
+	return response, d.call(response, "address_unused", map[string]interface{}{
+		"account_id": account,
+	})
+}
+
+func (d *Client) TransactionShow(txid string) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	return response, d.call(response, "transaction_show", map[string]interface{}{
+		"txid": txid,
+	})
+}
+
+func (d *Client) ChannelList(account *string, page uint64, pageSize uint64, wid *string) (*ChannelListResponse, error) {
+	if page == 0 {
+		return nil, errors.Err("pages start from 1")
+	}
+	response := new(ChannelListResponse)
+	return response, d.call(response, "channel_list", map[string]interface{}{
+		"account_id":       account,
+		"page":             page,
+		"page_size":        pageSize,
+		"include_protobuf": true,
+		"wallet_id":        wid,
+	})
+}
+
+type streamType string
+
+var (
+	StreamTypeVideo = streamType("video")
+	StreamTypeAudio = streamType("audio")
+	StreamTypeImage = streamType("image")
+)
+
+type Location struct {
+	Country    *string `json:"country,omitempty"`
+	State      *string `json:"state,omitempty"`
+	City       *string `json:"city,omitempty"`
+	PostalCode *string `json:"code,omitempty"`
+	Latitude   *string `json:"latitude,omitempty"`
+	Longitude  *string `json:"longitude,omitempty"`
+}
+type ClaimCreateOptions struct {
+	Title             *string    `json:"title,omitempty"`
+	Description       *string    `json:"description,omitempty"`
+	Tags              []string   `json:"tags,omitempty"`
+	Languages         []string   `json:"languages,omitempty"`
+	Locations         []Location `json:"locations,omitempty"`
+	ThumbnailURL      *string    `json:"thumbnail_url,omitempty"`
+	AccountID         *string    `json:"account_id,omitempty"`
+	ClaimAddress      *string    `json:"claim_address,omitempty"`
+	Preview           *bool      `json:"preview,omitempty"`
+	FundingAccountIDs []string   `json:"funding_account_ids,omitempty"`
+}
+
+type ChannelCreateOptions struct {
+	ClaimCreateOptions `json:",flatten"`
+	Email              *string  `json:"email,omitempty"`
+	WebsiteURL         *string  `json:"website_url,omitempty"`
+	CoverURL           *string  `json:"cover_url,omitempty"`
+	Featured           []string `json:"featured,omitempty"`
+	AccountID          *string  `json:"account_id,omitempty"`
+	FundingAccountIDs  []string `json:"funding_account_ids,omitempty"`
+}
+
+func (d *Client) ChannelCreate(name string, bid float64, options ChannelCreateOptions) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		Name                 string `json:"name"`
+		Bid                  string `json:"bid"`
+		FilePath             string `json:"file_path,omitempty"`
+		IncludeProtoBuf      bool   `json:"include_protobuf"`
+		ChannelCreateOptions `json:",flatten"`
+		Blocking             bool `json:"blocking"`
+	}{
+		Name:                 name,
+		Bid:                  fmt.Sprintf("%.6f", bid),
+		IncludeProtoBuf:      true,
+		ChannelCreateOptions: options,
+		Blocking:             true,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "channel_create", structs.Map(args))
+}
+
+type ChannelUpdateOptions struct {
+	ChannelCreateOptions `json:",flatten"`
+	NewSigningKey        *bool   `json:"new_signing_key,omitempty"`
+	ClearFeatured        *bool   `json:"clear_featured,omitempty"`
+	ClearTags            *bool   `json:"clear_tags,omitempty"`
+	ClearLanguages       *bool   `json:"clear_languages,omitempty"`
+	ClearLocations       *bool   `json:"clear_locations,omitempty"`
+	Bid                  *string `json:"bid,omitempty"`
+}
+
+func (d *Client) ChannelUpdate(claimID string, options ChannelUpdateOptions) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		ClaimID               string `json:"claim_id"`
+		IncludeProtoBuf       bool   `json:"include_protobuf"`
+		*ChannelUpdateOptions `json:",flatten"`
+		Blocking              bool `json:"blocking"`
+	}{
+		ClaimID:              claimID,
+		IncludeProtoBuf:      true,
+		ChannelUpdateOptions: &options,
+		Blocking:             true,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "channel_update", structs.Map(args))
+}
+
+type StreamCreateOptions struct {
+	ClaimCreateOptions `json:",flatten"`
+	Fee                *Fee        `json:",omitempty,flatten"`
+	Author             *string     `json:"author,omitempty"`
+	License            *string     `json:"license,omitempty"`
+	LicenseURL         *string     `json:"license_url,omitempty"`
+	StreamType         *streamType `json:"stream_type,omitempty"`
+	ReleaseTime        *int64      `json:"release_time,omitempty"`
+	Duration           *uint64     `json:"duration,omitempty"`
+	Width              *uint       `json:"width,omitempty"`
+	Height             *uint       `json:"height,omitempty"`
+	Preview            *string     `json:"preview,omitempty"`
+	AllowDuplicateName *bool       `json:"allow_duplicate_name,omitempty"`
+	ChannelName        *string     `json:"channel_name,omitempty"`
+	ChannelID          *string     `json:"channel_id,omitempty"`
+	ChannelAccountID   *string     `json:"channel_account_id,omitempty"`
+	AccountID          *string     `json:"account_id,omitempty"`
+}
+
+func (d *Client) StreamCreate(name, filePath string, bid float64, options StreamCreateOptions) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		Name                 string  `json:"name"`
+		Bid                  string  `json:"bid"`
+		FilePath             string  `json:"file_path,omitempty"`
+		FileSize             *string `json:"file_size,omitempty"`
+		IncludeProtoBuf      bool    `json:"include_protobuf"`
+		Blocking             bool    `json:"blocking"`
+		*StreamCreateOptions `json:",flatten"`
+	}{
+		Name:                name,
+		FilePath:            filePath,
+		Bid:                 fmt.Sprintf("%.6f", bid),
+		IncludeProtoBuf:     true,
+		Blocking:            true,
+		StreamCreateOptions: &options,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "stream_create", structs.Map(args))
+}
+
+func (d *Client) StreamAbandon(txID string, nOut uint64, accountID *string, blocking bool) (*ClaimAbandonResponse, error) {
+	response := new(ClaimAbandonResponse)
+	err := d.call(response, "stream_abandon", map[string]interface{}{
+		"txid":             txID,
+		"nout":             nOut,
+		"account_id":       accountID,
+		"include_protobuf": true,
+		"blocking":         true,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return response, nil
+}
+
+type StreamUpdateOptions struct {
+	ClearTags            *bool   `json:"clear_tags,omitempty"`
+	ClearLanguages       *bool   `json:"clear_languages,omitempty"`
+	ClearLocations       *bool   `json:"clear_locations,omitempty"`
+	Name                 *string `json:"name,omitempty"`
+	FilePath             *string `json:"file_path,omitempty"`
+	FileSize             *uint64 `json:"file_size,omitempty"`
+	Bid                  *string `json:"bid,omitempty"`
+	*StreamCreateOptions `json:",flatten"`
+}
+
+func (d *Client) StreamUpdate(claimID string, options StreamUpdateOptions) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		ClaimID              string `json:"claim_id"`
+		IncludeProtoBuf      bool   `json:"include_protobuf"`
+		*StreamUpdateOptions `json:",flatten"`
+		Blocking             bool `json:"blocking"`
+	}{
+		ClaimID:             claimID,
+		IncludeProtoBuf:     true,
+		StreamUpdateOptions: &options,
+		Blocking:            true,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "stream_update", structs.Map(args))
+}
+
+func (d *Client) ChannelAbandon(txID string, nOut uint64, accountID *string, blocking bool) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	err := d.call(response, "channel_abandon", map[string]interface{}{
+		"txid":             txID,
+		"nout":             nOut,
+		"account_id":       accountID,
+		"include_protobuf": true,
+		"blocking":         true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (d *Client) AddressList(account *string, address *string, page uint64, pageSize uint64) (*AddressListResponse, error) {
+	response := new(AddressListResponse)
+
+	args := struct {
+		AccountID *string `json:"account_id,omitempty"`
+		Address   *string `json:"address,omitempty"`
+		Page      uint64  `json:"page"`
+		PageSize  uint64  `json:"page_size"`
+	}{
+		AccountID: account,
+		Address:   address,
+		Page:      page,
+		PageSize:  pageSize,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "address_list", structs.Map(args))
+}
+
+func (d *Client) StreamList(account *string, page uint64, pageSize uint64) (*StreamListResponse, error) {
+	response := new(StreamListResponse)
+	err := d.call(response, "stream_list", map[string]interface{}{
+		"account_id":       account,
+		"include_protobuf": true,
+		"page":             page,
+		"page_size":        pageSize,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (d *Client) ClaimList(account *string, page uint64, pageSize uint64) (*ClaimListResponse, error) {
+	if page == 0 {
+		return nil, errors.Err("pages start from 1")
+	}
+	response := new(ClaimListResponse)
+	err := d.call(response, "claim_list", map[string]interface{}{
+		"account_id":       account,
+		"page":             page,
+		"page_size":        pageSize,
+		"include_protobuf": true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return response, nil
+}
+
+func (d *Client) Status() (*StatusResponse, error) {
+	response := new(StatusResponse)
+	return response, d.call(response, "status", map[string]interface{}{})
+}
+
+func (d *Client) TransactionList(account *string, wallet *string, page uint64, pageSize uint64) (*TransactionListResponse, error) {
+	response := new(TransactionListResponse)
+	return response, d.call(response, "transaction_list", map[string]interface{}{
+		"account_id": account,
+		"wallet_id":  wallet,
+		"page":       page,
+		"page_size":  pageSize,
+	})
+}
+
+func (d *Client) UTXOList(account *string, page uint64, pageSize uint64) (*UTXOListResponse, error) {
+	response := new(UTXOListResponse)
+	return response, d.call(response, "utxo_list", map[string]interface{}{
+		"account_id": account,
+		"page":       page,
+		"page_size":  pageSize,
+	})
+}
+
+func (d *Client) UTXORelease(account *string) (*UTXOReleaseResponse, error) {
+	response := new(UTXOReleaseResponse)
+	return response, d.call(response, "utxo_release", map[string]interface{}{
+		"account_id": account,
+	})
+}
+
+func (d *Client) Get(uri string) (*GetResponse, error) {
+	response := new(GetResponse)
+	return response, d.call(response, "get", map[string]interface{}{
+		"uri":              uri,
+		"include_protobuf": true,
+	})
+}
+
+func (d *Client) FileList(page uint64, pageSize uint64) (*FileListResponse, error) {
+	response := new(FileListResponse)
+	return response, d.call(response, "file_list", map[string]interface{}{
+		"include_protobuf": true,
+		"page":             page,
+		"page_size":        pageSize,
+	})
+}
+
+func (d *Client) Version() (*VersionResponse, error) {
+	response := new(VersionResponse)
+	return response, d.call(response, "version", map[string]interface{}{})
+}
+
+func (d *Client) Resolve(urls string) (*ResolveResponse, error) {
+	response := new(ResolveResponse)
+	return response, d.call(response, "resolve", map[string]interface{}{
+		"urls":             urls,
+		"include_protobuf": true,
+	})
+}
+
+type ClaimSearchArgs struct {
+	ClaimID               *string  `json:"claim_id,omitempty"`
+	TXID                  *string  `json:"txid,omitempty"`
+	Nout                  *uint    `json:"nout,omitempty"`
+	Name                  *string  `json:"name,omitempty"`
+	ClaimType             []string `json:"claim_type,omitempty"`
+	OrderBy               []string `json:"order_by,omitempty"`
+	LimitClaimsPerChannel *int     `json:"limit_claims_per_channel,omitempty"`
+	HasNoSource           *bool    `json:"has_no_source,omitempty"`
+	ReleaseTime           string   `json:"release_time,omitempty"`
+	ChannelIDs            []string `json:"channel_ids,omitempty"`
+	NoTotals              *bool    `json:"no_totals,omitempty"`
+	IncludeProtobuf       *bool    `json:"include_protobuf,omitempty"`
+	AnyTags               []string `json:"any_tags,omitempty"`
+
+	Page     uint64 `json:"page"`
+	PageSize uint64 `json:"page_size"`
+}
+
+func (d *Client) ClaimSearch(args ClaimSearchArgs) (*ClaimSearchResponse, error) {
+	response := new(ClaimSearchResponse)
+	if args.NoTotals == nil {
+		nototals := true
+		args.NoTotals = &nototals
+	}
+
+	if args.IncludeProtobuf == nil {
+		include := true
+		args.IncludeProtobuf = &include
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "claim_search", structs.Map(args))
+}
+
+func (d *Client) ChannelExport(channelClaimID string, channelName, accountID *string) (*ChannelExportResponse, error) {
+	response := new(ChannelExportResponse)
+	return response, d.call(response, "channel_export", map[string]interface{}{
+		"channel_id":   channelClaimID,
+		"channel_name": channelName,
+		"account_id":   accountID,
+	})
+}
+
+func (d *Client) ChannelImport(key string, walletID *string) (*ChannelImportResponse, error) {
+	response := new(ChannelImportResponse)
+	return response, d.call(response, "channel_import", map[string]interface{}{
+		"channel_data": key,
+		"wallet_id":    walletID,
+	})
+}
+
+func (d *Client) SupportList(accountID *string, page uint64, pageSize uint64) (*SupportListResponse, error) {
+	response := new(SupportListResponse)
+	return response, d.call(response, "support_list", map[string]interface{}{
+		"account_id": accountID,
+		"page":       page,
+		"page_size":  pageSize,
+	})
+}
+
+func (d *Client) SupportCreate(claimID string, amount string, tip *bool, accountID *string, fundingAccountIDs []string, walletID *string) (*TransactionSummary, error) {
+	response := new(TransactionSummary)
+	args := struct {
+		ClaimID           string   `json:"claim_id"`
+		Amount            string   `json:"amount"`
+		Tip               *bool    `json:"tip,omitempty"`
+		AccountID         *string  `json:"account_id,omitempty"`
+		FundingAccountIDs []string `json:"funding_account_ids,omitempty"`
+		Preview           bool     `json:"preview,omitempty"`
+		Blocking          bool     `json:"blocking,omitempty"`
+		WalletID          *string  `json:"wallet_id,omitempty"`
+	}{
+		ClaimID:           claimID,
+		AccountID:         accountID,
+		Blocking:          true,
+		Amount:            amount,
+		FundingAccountIDs: fundingAccountIDs,
+		Preview:           false,
+		Tip:               tip,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "support_create", structs.Map(args))
+}
+
+func (d *Client) SupportAbandon(claimID *string, txid *string, nout *uint, keep *string, accountID *string) (*TransactionSummary, error) {
+	if claimID == nil && (txid == nil || nout == nil) {
+		return nil, errors.Err("either claimID or txid+nout must be supplied")
+	}
+	response := new(TransactionSummary)
+	args := struct {
+		ClaimID   *string `json:"claim_id,omitempty"`
+		TxID      *string `json:"claim_id,omitempty"`
+		Nout      *uint   `json:"nout,omitempty"`
+		AccountID *string `json:"account_id,omitempty"`
+		Preview   bool    `json:"preview,omitempty"`
+		Blocking  bool    `json:"blocking,omitempty"`
+	}{
+		ClaimID:   claimID,
+		AccountID: accountID,
+		Nout:      nout,
+		TxID:      txid,
+		Blocking:  true,
+		Preview:   false,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "support_abandon", structs.Map(args))
+}
+
+func (d *Client) TxoSpend(txoType, claimID, txid, channelID, name, accountID *string) (*[]TransactionSummary, error) {
+	if txoType == nil && claimID == nil && txid == nil && channelID == nil && name == nil {
+		return nil, errors.Err("either txoType or claimID or channelID or name or txid must be supplied")
+	}
+	response := new([]TransactionSummary)
+	args := struct {
+		ClaimID       *string `json:"claim_id,omitempty"`
+		ChannelID     *string `json:"channel_id,omitempty"`
+		Name          *string `json:"name,omitempty"`
+		TxID          *string `json:"claim_id,omitempty"`
+		Type          *string `json:"type,omitempty"`
+		AccountID     *string `json:"account_id,omitempty"`
+		Preview       bool    `json:"preview,omitempty"`
+		Blocking      bool    `json:"blocking,omitempty"`
+		IncludeFullTx bool    `json:"include_full_tx,omitempty"`
+	}{
+		ClaimID:       claimID,
+		ChannelID:     channelID,
+		Name:          name,
+		Type:          txoType,
+		AccountID:     accountID,
+		TxID:          txid,
+		Blocking:      true,
+		Preview:       false,
+		IncludeFullTx: true,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "txo_spend", structs.Map(args))
+}
+
+func (d *Client) AccountAdd(accountName string, seed *string, privateKey *string, publicKey *string, singleKey *bool, walletID *string) (*Account, error) {
+	response := new(Account)
+
+	args := struct {
+		AccountName string  `json:"account_name"`
+		Seed        *string `json:"seed,omitempty"`
+		PrivateKey  *string `json:"private_key,omitempty"`
+		PublicKey   *string `json:"public_key,omitempty"`
+		SingleKey   *bool   `json:"single_key,omitempty"`
+		WalletID    *string `json:"wallet_id,omitempty"`
+	}{
+		AccountName: accountName,
+		Seed:        seed,
+		PrivateKey:  privateKey,
+		PublicKey:   publicKey,
+		SingleKey:   singleKey,
+		WalletID:    walletID,
+	}
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "account_add", structs.Map(args))
+}
+
+type WalletCreateOpts struct {
+	ID            string `json:"wallet_id"`
+	SkipOnStartup bool   `json:"skip_on_startup,omitempty"`
+	CreateAccount bool   `json:"create_account,omitempty"`
+	SingleKey     bool   `json:"single_key,omitempty"`
+}
+
+func (d *Client) WalletCreate(id string, opts *WalletCreateOpts) (*Wallet, error) {
+	response := new(Wallet)
+	if opts == nil {
+		opts = &WalletCreateOpts{}
+	}
+	opts.ID = id
+	structs.DefaultTagName = "json"
+	return response, d.call(response, "wallet_create", structs.Map(opts))
+}
+
+func (d *Client) WalletAdd(id string) (*Wallet, error) {
+	response := new(Wallet)
+	return response, d.call(response, "wallet_add", map[string]interface{}{"wallet_id": id})
+}
+
+func (d *Client) WalletList(id string, page uint64, pageSize uint64) (*WalletList, error) {
+	response := new(WalletList)
+	params := map[string]interface {
+	}{
+		"page":      page,
+		"page_size": pageSize,
+	}
+	if id != "" {
+		params["wallet_id"] = id
+	}
+	return response, d.call(response, "wallet_list", params)
+}
+
+func (d *Client) WalletRemove(id string) (*Wallet, error) {
+	response := new(Wallet)
+	return response, d.call(response, "wallet_remove", map[string]interface{}{"wallet_id": id})
+}

extras/jsonrpc/daemon_test.go

@@ -0,0 +1,845 @@
+package jsonrpc
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"os"
+	"strconv"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/shopspring/decimal"
+	"github.com/stretchr/testify/assert"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"github.com/lbryio/lbry.go/v2/extras/util"
+)
+
+func prettyPrint(i interface{}) {
+	s, _ := json.MarshalIndent(i, "", "\t")
+	fmt.Println(string(s))
+}
+
+func TestMain(m *testing.M) {
+	rand.Seed(time.Now().UnixNano())
+	code := m.Run()
+	os.Exit(code)
+}
+
+func TestClient_AccountFund(t *testing.T) {
+	d := NewClient("")
+	accounts, err := d.AccountList(1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	account := (accounts.Items)[0].ID
+	balanceString, err := d.AccountBalance(&account)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	balance, err := strconv.ParseFloat(balanceString.Available.String(), 64)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	got, err := d.AccountFund(account, account, fmt.Sprintf("%f", balance/2.0), 40, false)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_AccountSend(t *testing.T) {
+	d := NewClient("")
+	accounts, err := d.AccountList(1, 20)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.NotEmpty(t, accounts.Items[1].ID) {
+		return
+	}
+	account := (accounts.Items)[1].ID
+
+	addressess, err := d.AddressList(&account, nil, 1, 20)
+	if !assert.NoError(t, err) {
+		return
+	}
+	if !assert.NotEmpty(t, addressess.Items) {
+		return
+	}
+
+	got, err := d.AccountSend(&account, "0.01", string(addressess.Items[0].Address))
+	if !assert.NoError(t, err) {
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_AccountList(t *testing.T) {
+	d := NewClient("")
+	got, err := d.AccountList(1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_SingleAccountList(t *testing.T) {
+	d := NewClient("")
+	name := "test" + fmt.Sprintf("%d", rand.Int()) + "@lbry.com"
+	createdAccount, err := d.AccountCreate(name, false)
+	if err != nil {
+		t.Fatal(err)
+	}
+	account, err := d.SingleAccountList(createdAccount.ID)
+	if err != nil {
+		t.Error(err)
+	}
+	prettyPrint(*createdAccount)
+	prettyPrint(*account)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if account.Items[0].Name != name {
+		t.Fatalf("account name mismatch: %v != %v", account.Items[0].Name, name)
+	}
+}
+
+func TestClient_AccountBalance(t *testing.T) {
+	d := NewClient("")
+	got, err := d.AccountBalance(nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_AddressUnused(t *testing.T) {
+	d := NewClient("")
+	got, err := d.AddressUnused(nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_ChannelList(t *testing.T) {
+	d := NewClient("")
+	got, err := d.ChannelList(nil, 1, 50, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+var channelID string
+
+func TestClient_ChannelCreate(t *testing.T) {
+	d := NewClient("")
+	got, err := d.ChannelCreate("@Test"+fmt.Sprintf("%d", time.Now().Unix()), 1.337, ChannelCreateOptions{
+		ClaimCreateOptions: ClaimCreateOptions{
+			Title:       util.PtrToString("Mess with the channels"),
+			Description: util.PtrToString("And you'll get what you deserve"),
+			Tags:        []string{"we", "got", "tags"},
+			Languages:   []string{"en-US"},
+			Locations: []Location{{
+				Country: util.PtrToString("CH"),
+				State:   util.PtrToString("Ticino"),
+				City:    util.PtrToString("Lugano"),
+			}},
+			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2022-06-10_17-18-29-409175881.png"),
+		},
+		Email:      util.PtrToString("niko@lbry.com"),
+		WebsiteURL: util.PtrToString("https://lbry.com"),
+		CoverURL:   util.PtrToString("https://scrn.storni.info/2022-06-10_17-18-29-409175881.png"),
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	channelID = got.Outputs[0].ClaimID
+	prettyPrint(*got)
+}
+
+func TestClient_StreamCreate(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	addressResponse, err := d.AddressUnused(nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	address := string(*addressResponse)
+	f, e := os.OpenFile("/tmp/test.txt", os.O_RDONLY|os.O_CREATE, 0666)
+	if e != nil {
+		t.Error(e)
+		return
+	}
+
+	_, _ = f.WriteString("test")
+	got, err := d.StreamCreate("test"+fmt.Sprintf("%d", time.Now().Unix()), "/tmp/test.txt", 1.437, StreamCreateOptions{
+		ClaimCreateOptions: ClaimCreateOptions{
+			Title:       util.PtrToString("This is a Test Title" + fmt.Sprintf("%d", time.Now().Unix())),
+			Description: util.PtrToString("My Special Description"),
+			Tags:        []string{"nsfw", "test"},
+			Languages:   []string{"en-US", "fr-CH"},
+			Locations: []Location{{
+				Country:    util.PtrToString("CH"),
+				State:      util.PtrToString("Ticino"),
+				City:       util.PtrToString("Lugano"),
+				PostalCode: util.PtrToString("6900"),
+				Latitude:   nil,
+				Longitude:  nil,
+			}},
+			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2019-01-18_16-37-39-098537783.png"),
+			AccountID:    nil,
+			ClaimAddress: &address,
+			Preview:      nil,
+		},
+
+		Fee: &Fee{
+			FeeCurrency: "LBC",
+			FeeAmount:   decimal.NewFromFloat(1.0),
+			FeeAddress:  &address,
+		},
+		Author:             util.PtrToString("Niko"),
+		License:            util.PtrToString("FREE"),
+		LicenseURL:         nil,
+		ReleaseTime:        nil,
+		Duration:           nil,
+		Preview:            nil,
+		AllowDuplicateName: nil,
+		ChannelName:        nil,
+		ChannelID:          util.PtrToString(channelID),
+		ChannelAccountID:   nil,
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_ChannelUpdate(t *testing.T) {
+	d := NewClient("")
+	got, err := d.ChannelUpdate(channelID, ChannelUpdateOptions{
+		Bid:            util.PtrToString("0.01"),
+		ClearLanguages: util.PtrToBool(true),
+		ClearLocations: util.PtrToBool(true),
+		ClearTags:      util.PtrToBool(true),
+		ChannelCreateOptions: ChannelCreateOptions{
+			ClaimCreateOptions: ClaimCreateOptions{
+				Title:       util.PtrToString("Mess with the channels"),
+				Description: util.PtrToString("And you'll get what you deserve"),
+				Tags:        []string{"we", "got", "more", "tags"},
+				Languages:   []string{"en-US"},
+				Locations: []Location{{
+					Country: util.PtrToString("CH"),
+					State:   util.PtrToString("Ticino"),
+					City:    util.PtrToString("Lugano"),
+				}},
+				ThumbnailURL: util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+			},
+			Email:      util.PtrToString("niko@lbry.com"),
+			WebsiteURL: util.PtrToString("https://lbry.com"),
+			CoverURL:   util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+		}})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_ChannelAbandon(t *testing.T) {
+	d := NewClient("")
+	channelName := "@TestToDelete" + fmt.Sprintf("%d", time.Now().Unix())
+	channelResponse, err := d.ChannelCreate(channelName, 13.37, ChannelCreateOptions{
+		ClaimCreateOptions: ClaimCreateOptions{
+			Title:       util.PtrToString("Mess with the channels"),
+			Description: util.PtrToString("And you'll get what you deserve"),
+			Tags:        []string{"we", "got", "tags"},
+			Languages:   []string{"en-US"},
+			Locations: []Location{{
+				Country: util.PtrToString("CH"),
+				State:   util.PtrToString("Ticino"),
+				City:    util.PtrToString("Lugano"),
+			}},
+			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+		},
+		Email:      util.PtrToString("niko@lbry.com"),
+		WebsiteURL: util.PtrToString("https://lbry.com"),
+		CoverURL:   util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	txID := channelResponse.Outputs[0].Txid
+	nout := channelResponse.Outputs[0].Nout
+	time.Sleep(10 * time.Second)
+	got, err := d.ChannelAbandon(txID, nout, nil, false)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_AddressList(t *testing.T) {
+	d := NewClient("")
+	got, err := d.AddressList(nil, nil, 1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_ClaimList(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.ClaimList(nil, 1, 10)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_StreamList(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.StreamList(nil, 1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_TransactionList(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.TransactionList(nil, nil, 1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_SupportTest(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.ChannelCreate("@Test"+fmt.Sprintf("%d", time.Now().Unix()), 13.37, ChannelCreateOptions{
+		ClaimCreateOptions: ClaimCreateOptions{
+			Title:       util.PtrToString("Mess with the channels"),
+			Description: util.PtrToString("And you'll get what you deserve"),
+			Tags:        []string{"we", "got", "tags"},
+			Languages:   []string{"en-US"},
+			Locations: []Location{{
+				Country: util.PtrToString("CH"),
+				State:   util.PtrToString("Ticino"),
+				City:    util.PtrToString("Lugano"),
+			}},
+			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+		},
+		Email:      util.PtrToString("niko@lbry.com"),
+		WebsiteURL: util.PtrToString("https://lbry.com"),
+		CoverURL:   util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	time.Sleep(10 * time.Second)
+	got2, err := d.SupportCreate(got.Outputs[0].ClaimID, "1.0", util.PtrToBool(true), nil, nil, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got2)
+
+	got3, err := d.SupportList(nil, 1, 10)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	found := false
+	for _, support := range got3.Items {
+		if support.ClaimID == got.Outputs[0].ClaimID {
+			found = true
+		}
+	}
+	if !found {
+		t.Error(errors.Err("support not found"))
+		return
+	}
+	prettyPrint(*got3)
+	got4, err := d.SupportAbandon(util.PtrToString(got.Outputs[0].ClaimID), nil, nil, nil, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got4)
+}
+
+func TestClient_TxoSpendTest(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.ChannelCreate("@Test"+fmt.Sprintf("%d", time.Now().Unix()), 13.37, ChannelCreateOptions{
+		ClaimCreateOptions: ClaimCreateOptions{
+			Title:       util.PtrToString("Mess with the channels"),
+			Description: util.PtrToString("And you'll get what you deserve"),
+			Tags:        []string{"we", "got", "tags"},
+			Languages:   []string{"en-US"},
+			Locations: []Location{{
+				Country: util.PtrToString("CH"),
+				State:   util.PtrToString("Ticino"),
+				City:    util.PtrToString("Lugano"),
+			}},
+			ThumbnailURL: util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+		},
+		Email:      util.PtrToString("niko@lbry.com"),
+		WebsiteURL: util.PtrToString("https://lbry.com"),
+		CoverURL:   util.PtrToString("https://scrn.storni.info/2019-04-12_15-43-25-001592625.png"),
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	time.Sleep(10 * time.Second)
+	got2, err := d.SupportCreate(got.Outputs[0].ClaimID, "1.0", util.PtrToBool(true), nil, nil, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got2)
+
+	got3, err := d.SupportList(nil, 1, 10)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	found := false
+	for _, support := range got3.Items {
+		if support.ClaimID == got.Outputs[0].ClaimID {
+			found = true
+		}
+	}
+	if !found {
+		t.Error(errors.Err("support not found"))
+		return
+	}
+	prettyPrint(*got3)
+	got4, err := d.TxoSpend(util.PtrToString("support"), util.PtrToString(got.Outputs[0].ClaimID), nil, nil, nil, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got4)
+	time.Sleep(10 * time.Second)
+	got3, err = d.SupportList(nil, 1, 10)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	found = false
+	for _, support := range got3.Items {
+		if support.ClaimID == got.Outputs[0].ClaimID {
+			found = true
+		}
+	}
+	if found {
+		t.Error(errors.Err("support found even though it should have been abandoned"))
+		return
+	}
+	prettyPrint(*got3)
+	got4, err = d.TxoSpend(util.PtrToString("channel"), util.PtrToString(got.Outputs[0].ClaimID), nil, nil, nil, nil)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got4)
+	time.Sleep(10 * time.Second)
+
+	got5, err := d.ClaimList(nil, 1, 50)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	for _, claim := range got5.Claims {
+		if claim.ClaimID == got.Outputs[0].ClaimID {
+			t.Error(errors.Err("claim found even though it should have been abandoned"))
+			return
+		}
+	}
+	prettyPrint(*got5)
+}
+
+func TestClient_ClaimSearch(t *testing.T) {
+	d := NewClient("")
+	got, err := d.ClaimSearch(ClaimSearchArgs{
+		ChannelIDs:  []string{channelID},
+		ReleaseTime: ">1633350820",
+		HasNoSource: util.PtrToBool(true),
+		OrderBy:     []string{"^release_time"},
+		Page:        1,
+		PageSize:    20,
+	})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_Status(t *testing.T) {
+	d := NewClient("")
+	got, err := d.Status()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_UTXOList(t *testing.T) {
+	d := NewClient("")
+	got, err := d.UTXOList(nil, 1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_Version(t *testing.T) {
+	d := NewClient("")
+	got, err := d.Version()
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_GetFile(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.Get("lbry://test1559058649")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_FileList(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.FileList(1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_Resolve(t *testing.T) {
+	_ = os.Setenv("BLOCKCHAIN_NAME", "lbrycrd_regtest")
+	d := NewClient("")
+	got, err := d.Resolve("test1559058649")
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_AccountSet(t *testing.T) {
+	d := NewClient("")
+	accounts, err := d.AccountList(1, 20)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	account := (accounts.Items)[0].ID
+
+	got, err := d.AccountSet(account, AccountSettings{ChangeMaxUses: util.PtrToInt(10000)})
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	prettyPrint(*got)
+}
+
+func TestClient_AccountCreate(t *testing.T) {
+	d := NewClient("")
+	name := "lbry#user#id:" + fmt.Sprintf("%d", rand.Int())
+	account, err := d.AccountCreate(name, false)
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+	if account.Name != name {
+		t.Errorf("account name mismatch, expected %q, got %q", name, account.Name)
+		return
+	}
+	prettyPrint(*account)
+}
+
+func TestClient_AccountAdd(t *testing.T) {
+	d := NewClient("")
+	name := "test" + fmt.Sprintf("%d", time.Now().Unix()) + "@lbry.com"
+	pubKey := "tpubDA9GDAntyJu4hD3wU7175p7CuV6DWbYXfyb2HedBA3yuBp9HZ4n3QE4Ex6RHCSiEuVp2nKAL1Lzf2ZLo9ApaFgNaJjG6Xo1wB3iEeVbrDZp"
+	account, err := d.AccountAdd(name, nil, nil, &pubKey, util.PtrToBool(true), nil)
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+	if account.Name != name {
+		t.Errorf("account name mismatch, expected %q, got %q", name, account.Name)
+		return
+	}
+	if account.PublicKey != pubKey {
+		t.Errorf("public key mismatch, expected %q, got %q", name, account.Name)
+		return
+	}
+	prettyPrint(*account)
+}
+
+func TestClient_AccountRemove(t *testing.T) {
+	d := NewClient("")
+	name := "lbry#user#id:" + fmt.Sprintf("%d", rand.Int())
+	createdAccount, err := d.AccountCreate(name, false)
+	if err != nil {
+		t.Fatal(err)
+		return
+	}
+	removedAccount, err := d.AccountRemove(createdAccount.ID)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+	if removedAccount.ID != createdAccount.ID {
+		t.Error("accounts IDs mismatch")
+	}
+
+	account, err := d.SingleAccountList(createdAccount.ID)
+	if err != nil {
+		if strings.Contains(err.Error(), "Couldn't find account:") {
+			prettyPrint(*removedAccount)
+			return
+		}
+		t.Fatal(err)
+	}
+	t.Error("account was not removed")
+	prettyPrint(*account)
+}
+
+func TestClient_ChannelExport(t *testing.T) {
+	d := NewClient("")
+	response, err := d.ChannelExport(channelID, nil, nil)
+	if err != nil {
+		t.Error(err)
+	}
+	if response == nil || len(*response) == 0 {
+		t.Error("nothing returned!")
+	}
+	t.Log("Export:", *response)
+}
+
+func TestClient_ChannelImport(t *testing.T) {
+	d := NewClient("")
+
+	// A channel created just for automated testing purposes
+	channelName := "@LbryAutomatedTestChannel"
+	channelkey := "7943FWPBHZES4dUcMXSpDYwoM5a2tsyJT1R8V54QoUhekGcqmeH3hbzDXoLLQ8" +
+		"oKkfb99PgGK5efrZeYqaxg4X5XRJMJ6gKC8hqKcnwhYkmKDXmoBDNgd2ccZ9jhP8z" +
+		"HG3NJorAN9Hh4XMyBc5goBLZYYvC9MYvBmT3Fcteb5saqMvmQxFURv74NqXLQZC1t" +
+		"p6iRZKfTj77Pd5gsBsCYAbVmCqzbm5m1hHkUmfFEZVGcQNTYCDwZn543xSMYvSPnJ" +
+		"zt8tRYCJWaPdj713uENZZMo3gxuAMb1NwSnx8tbwETp7WPkpFLL6HZ9jKpB8BURHM" +
+		"F1RFD1PRyqbC6YezPyPQ2oninKKHdBduvXZG5KF2G2Q3ixsuE2ntifBBo1f5PotRk" +
+		"UanXKEafWxvXAayJjpsmZ4bFt7n6Xg4438WZXBiZKCPobLJAiHfe72n618kE6PCNU" +
+		"77cyU5Rk8J3CuY6QzZPzwuiXz2GLfkUMCYd9jGT6g53XbE6SwCsmGnd9NJkBAaJf5" +
+		"1FAYRURrhHnp79PAoHftEWtZEuU8MCPMdSRjzxYMRS4ScUzg5viDMTAkE8frsfCVZ" +
+		"hxsFwGUyNNno8eiqrrYmpbJGEwwK3S4437JboAUEFPdMNn8zNQWZcLLVrK9KyQeKM" +
+		"XpKkf4zJV6sZJ7gBMpzvPL18ULEgXTy7VsNBKmsfC1rM4WVG9ri1UixEcLDS79foC" +
+		"Jb3FnSr1T4MRKESeN3W"
+	response, err := d.ChannelImport(channelkey, nil)
+	if err != nil {
+		t.Error(err)
+	}
+	channels, err := d.ChannelList(nil, 1, 50, nil)
+	if err != nil {
+		t.Error(err)
+	}
+	seen := false
+	for _, c := range channels.Items {
+		if c.Name == channelName {
+			seen = true
+		}
+	}
+	if !seen {
+		t.Error("couldn't find imported channel")
+	}
+	t.Log("Response:", *response)
+}
+
+func TestClient_ChannelImportWithWalletID(t *testing.T) {
+	d := NewClient("")
+
+	id := "lbry#wallet#id:" + fmt.Sprintf("%d", rand.Int())
+	wallet, err := d.WalletCreate(id, nil)
+	if err != nil {
+		t.Error(err)
+	}
+	// A channel created just for automated testing purposes
+	channelName := "@LbryAutomatedTestChannel"
+	channelKey := "7943FWPBHZES4dUcMXSpDYwoM5a2tsyJT1R8V54QoUhekGcqmeH3hbzDXoLLQ8" +
+		"oKkfb99PgGK5efrZeYqaxg4X5XRJMJ6gKC8hqKcnwhYkmKDXmoBDNgd2ccZ9jhP8z" +
+		"HG3NJorAN9Hh4XMyBc5goBLZYYvC9MYvBmT3Fcteb5saqMvmQxFURv74NqXLQZC1t" +
+		"p6iRZKfTj77Pd5gsBsCYAbVmCqzbm5m1hHkUmfFEZVGcQNTYCDwZn543xSMYvSPnJ" +
+		"zt8tRYCJWaPdj713uENZZMo3gxuAMb1NwSnx8tbwETp7WPkpFLL6HZ9jKpB8BURHM" +
+		"F1RFD1PRyqbC6YezPyPQ2oninKKHdBduvXZG5KF2G2Q3ixsuE2ntifBBo1f5PotRk" +
+		"UanXKEafWxvXAayJjpsmZ4bFt7n6Xg4438WZXBiZKCPobLJAiHfe72n618kE6PCNU" +
+		"77cyU5Rk8J3CuY6QzZPzwuiXz2GLfkUMCYd9jGT6g53XbE6SwCsmGnd9NJkBAaJf5" +
+		"1FAYRURrhHnp79PAoHftEWtZEuU8MCPMdSRjzxYMRS4ScUzg5viDMTAkE8frsfCVZ" +
+		"hxsFwGUyNNno8eiqrrYmpbJGEwwK3S4437JboAUEFPdMNn8zNQWZcLLVrK9KyQeKM" +
+		"XpKkf4zJV6sZJ7gBMpzvPL18ULEgXTy7VsNBKmsfC1rM4WVG9ri1UixEcLDS79foC" +
+		"Jb3FnSr1T4MRKESeN3W"
+	response, err := d.ChannelImport(channelKey, &wallet.ID)
+	if err != nil {
+		t.Error(err)
+	}
+	channels, err := d.ChannelList(nil, 1, 50, &wallet.ID)
+	if err != nil {
+		t.Error(err)
+	}
+	seen := false
+	for _, c := range channels.Items {
+		if c.Name == channelName {
+			seen = true
+		}
+	}
+	if !seen {
+		t.Error("couldn't find imported channel")
+	}
+	t.Log("Response:", *response)
+}
+
+func TestClient_WalletCreate(t *testing.T) {
+	d := NewClient("")
+
+	id := "lbry#wallet#id:" + fmt.Sprintf("%d", rand.Int())
+	wallet, err := d.WalletCreate(id, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if wallet.ID != id {
+		prettyPrint(*wallet)
+		t.Fatalf("wallet ID mismatch, expected %q, got %q", id, wallet.Name)
+	}
+}
+
+func TestClient_WalletCreateWithOpts(t *testing.T) {
+	d := NewClient("")
+
+	id := "lbry#wallet#id:" + fmt.Sprintf("%d", rand.Int())
+	wallet, err := d.WalletCreate(id, &WalletCreateOpts{CreateAccount: true, SingleKey: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+	accounts, err := d.AccountListForWallet(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	prettyPrint(wallet)
+	prettyPrint(accounts)
+	if accounts.Items[0].Name == "" {
+		t.Fatalf("account name is empty")
+	}
+}
+
+func TestClient_WalletList(t *testing.T) {
+	d := NewClient("")
+
+	id := "lbry#wallet#id:" + fmt.Sprintf("%d", rand.Int())
+	_, err := d.WalletList(id, 1, 20)
+	if err == nil {
+		t.Fatalf("wallet %v was unexpectedly found", id)
+	}
+	derr, ok := err.(Error)
+	if !ok {
+		t.Fatalf("unknown error returned: %s", err)
+	}
+	if derr.Name != ErrorWalletNotLoaded {
+		t.Fatal(err)
+	}
+
+	_, err = d.WalletCreate(id, &WalletCreateOpts{CreateAccount: true, SingleKey: true})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	wList, err := d.WalletList(id, 1, 20)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(wList.Items) < 1 {
+		t.Fatal("wallet list is empty")
+	}
+	if (wList.Items)[0].ID != id {
+		t.Fatalf("wallet ID mismatch, expected %q, got %q", id, (wList.Items)[0].ID)
+	}
+}
+
+func TestClient_WalletRemoveWalletAdd(t *testing.T) {
+	d := NewClient("")
+
+	id := "lbry#wallet#id:" + fmt.Sprintf("%d", rand.Int())
+	wallet, err := d.WalletCreate(id, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	_, err = d.WalletRemove(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	addedWallet, err := d.WalletAdd(id)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if addedWallet.ID != wallet.ID {
+		prettyPrint(*addedWallet)
+		t.Fatalf("wallet ID mismatch, expected %q, got %q", wallet.ID, addedWallet.Name)
+	}
+}
+
+func TestClient_TransactionSummary(t *testing.T) {
+	d := NewClient("https://api.na-backend.odysee.com/api/v1/proxy")
+	r, err := d.TransactionShow("d104a1616c6af581e2046819de678f370d624e97cf176f95acaec4b183a42db6")
+	if err != nil {
+		t.Error(err)
+	}
+	if len(r.Outputs) != 2 {
+		t.Fatal("found wrong transaction")
+	}
+	if r.Outputs[0].Amount != "5.0" {
+		t.Error("found wrong lbc amount for transaction.")
+	}
+}

extras/jsonrpc/daemon_types.go

@@ -0,0 +1,670 @@
+package jsonrpc
+
+import (
+	"encoding/hex"
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"reflect"
+	"strings"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/stream"
+
+	schema "github.com/lbryio/lbry.go/v2/schema/stake"
+	lbryschema "github.com/lbryio/types/v2/go"
+
+	"github.com/shopspring/decimal"
+)
+
+type Currency string
+
+const (
+	CurrencyLBC = Currency("LBC")
+	CurrencyUSD = Currency("USD")
+	CurrencyBTC = Currency("BTC")
+)
+
+type Fee struct {
+	FeeCurrency Currency        `json:"fee_currency"`
+	FeeAmount   decimal.Decimal `json:"fee_amount"`
+	FeeAddress  *string         `json:"fee_address"`
+}
+
+type File struct {
+	AddedOn              int64             `json:"added_on"`
+	BlobsCompleted       uint64            `json:"blobs_completed"`
+	BlobsInStream        uint64            `json:"blobs_in_stream"`
+	BlobsRemaining       uint64            `json:"blobs_remaining"`
+	ChannelClaimID       string            `json:"channel_claim_id"`
+	ChannelName          string            `json:"channel_name"`
+	ClaimID              string            `json:"claim_id"`
+	ClaimName            string            `json:"claim_name"`
+	Completed            bool              `json:"completed"`
+	Confirmations        int64             `json:"confirmations"`
+	ContentFee           *Fee              `json:"content_fee"`
+	DownloadDirectory    string            `json:"download_directory"`
+	DownloadPath         string            `json:"download_path"`
+	FileName             string            `json:"file_name"`
+	Height               int               `json:"height"`
+	IsFullyReflected     bool              `json:"is_fully_reflected"`
+	Key                  string            `json:"key"`
+	Value                *lbryschema.Claim `json:"protobuf"`
+	MimeType             string            `json:"mime_type"`
+	Nout                 int               `json:"nout"`
+	Outpoint             string            `json:"outpoint"`
+	PurchaseReceipt      interface{}       `json:"purchase_receipt"`
+	ReflectorProgress    int               `json:"reflector_progress"`
+	SdHash               string            `json:"sd_hash"`
+	Status               string            `json:"status"`
+	Stopped              bool              `json:"stopped"`
+	StreamHash           string            `json:"stream_hash"`
+	StreamName           string            `json:"stream_name"`
+	StreamingURL         string            `json:"streaming_url"`
+	SuggestedFileName    string            `json:"suggested_file_name"`
+	Timestamp            int64             `json:"timestamp"`
+	TotalBytes           uint64            `json:"total_bytes"`
+	TotalBytesLowerBound uint64            `json:"total_bytes_lower_bound"`
+	Txid                 string            `json:"txid"`
+	UploadingToReflector bool              `json:"uploading_to_reflector"`
+	WrittenBytes         uint64            `json:"written_bytes"`
+}
+
+func getEnumVal(enum map[string]int32, data interface{}) (int32, error) {
+	s, ok := data.(string)
+	if !ok {
+		return 0, errors.Err("expected a string")
+	}
+	val, ok := enum[s]
+	if !ok {
+		return 0, errors.Err("invalid enum key")
+	}
+	return val, nil
+}
+
+func fixDecodeProto(src, dest reflect.Type, data interface{}) (interface{}, error) {
+	switch dest {
+	case reflect.TypeOf(uint64(0)):
+		if n, ok := data.(json.Number); ok {
+			val, err := n.Int64()
+			if err != nil {
+				return nil, errors.Wrap(err, 0)
+			} else if val < 0 {
+				return nil, errors.Err("must be unsigned int")
+			}
+			return uint64(val), nil
+		}
+	case reflect.TypeOf([]byte{}):
+		if s, ok := data.(string); ok {
+			return []byte(s), nil
+		}
+
+	case reflect.TypeOf(decimal.Decimal{}):
+		if n, ok := data.(json.Number); ok {
+			val, err := n.Float64()
+			if err != nil {
+				return nil, errors.Wrap(err, 0)
+			}
+			return decimal.NewFromFloat(val), nil
+		} else if s, ok := data.(string); ok {
+			d, err := decimal.NewFromString(s)
+			if err != nil {
+				return nil, errors.Wrap(err, 0)
+			}
+			return d, nil
+		}
+
+	case reflect.TypeOf(lbryschema.Fee_Currency(0)):
+		val, err := getEnumVal(lbryschema.Fee_Currency_value, data)
+		return lbryschema.Fee_Currency(val), err
+	case reflect.TypeOf(lbryschema.Claim{}):
+		blockChainName := os.Getenv("BLOCKCHAIN_NAME")
+		if blockChainName == "" {
+			blockChainName = "lbrycrd_main"
+		}
+
+		claim, err := schema.DecodeClaimHex(data.(string), blockChainName)
+		if err != nil {
+			return nil, err
+		}
+		return claim.Claim, nil
+	}
+
+	return data, nil
+}
+
+type WalletBalanceResponse decimal.Decimal
+
+type PeerListResponsePeer struct {
+	IP     string `json:"host"`
+	Port   uint   `json:"port"`
+	NodeId string `json:"node_id"`
+}
+type PeerListResponse []PeerListResponsePeer
+
+type BlobGetResponse struct {
+	Blobs []struct {
+		BlobHash string `json:"blob_hash,omitempty"`
+		BlobNum  int    `json:"blob_num"`
+		IV       string `json:"iv"`
+		Length   int    `json:"length"`
+	} `json:"blobs"`
+	Key               string `json:"key"`
+	StreamHash        string `json:"stream_hash"`
+	StreamName        string `json:"stream_name"`
+	StreamType        string `json:"stream_type"`
+	SuggestedFileName string `json:"suggested_file_name"`
+}
+
+type StreamCostEstimateResponse decimal.Decimal
+
+type BlobAvailability struct {
+	IsAvailable      bool     `json:"is_available"`
+	ReachablePeers   []string `json:"reachable_peers"`
+	UnReachablePeers []string `json:"unreachable_peers"`
+}
+
+type StreamAvailabilityResponse struct {
+	IsAvailable          bool             `json:"is_available"`
+	DidDecode            bool             `json:"did_decode"`
+	DidResolve           bool             `json:"did_resolve"`
+	IsStream             bool             `json:"is_stream"`
+	NumBlobsInStream     uint64           `json:"num_blobs_in_stream"`
+	SDHash               string           `json:"sd_hash"`
+	SDBlobAvailability   BlobAvailability `json:"sd_blob_availability"`
+	HeadBlobHash         string           `json:"head_blob_hash"`
+	HeadBlobAvailability BlobAvailability `json:"head_blob_availability"`
+	UseUPNP              bool             `json:"use_upnp"`
+	UPNPRedirectIsSet    bool             `json:"upnp_redirect_is_set"`
+	Error                string           `json:"error,omitempty"`
+}
+
+type GetResponse File
+type FileListResponse struct {
+	Items      []File `json:"items"`
+	Page       uint64 `json:"page"`
+	PageSize   uint64 `json:"page_size"`
+	TotalPages uint64 `json:"total_pages"`
+}
+
+type Account struct {
+	AddressGenerator struct {
+		Change struct {
+			Gap                   uint64 `json:"gap"`
+			MaximumUsesPerAddress uint64 `json:"maximum_uses_per_address"`
+		} `json:"change"`
+		Name      string `json:"name"`
+		Receiving struct {
+			Gap                   uint64 `json:"gap"`
+			MaximumUsesPerAddress uint64 `json:"maximum_uses_per_address"`
+		} `json:"receiving"`
+	} `json:"address_generator"`
+	Certificates uint64   `json:"certificates"`
+	Coins        float64  `json:"coins"`
+	Encrypted    bool     `json:"encrypted"`
+	ID           string   `json:"id"`
+	IsDefault    bool     `json:"is_default"`
+	Ledger       *string  `json:"ledger,omitempty"`
+	ModifiedOn   *float64 `json:"modified_on,omitempty"`
+	Name         string   `json:"name"`
+	Preferences  *struct {
+		Theme string `json:"theme"`
+	} `json:"preferences,omitempty"`
+	PrivateKey *string `json:"private_key,omitempty"`
+	PublicKey  string  `json:"public_key"`
+	Seed       *string `json:"seed,omitempty"`
+	Satoshis   uint64  `json:"satoshis"`
+}
+
+type AccountListResponse struct {
+	Items      []Account `json:"items"`
+	Page       uint64    `json:"page"`
+	PageSize   uint64    `json:"page_size"`
+	TotalPages uint64    `json:"total_pages"`
+}
+
+type AccountBalanceResponse struct {
+	Available         decimal.Decimal `json:"available"`
+	Reserved          decimal.Decimal `json:"reserved"`
+	ReservedSubtotals struct {
+		Claims   decimal.Decimal `json:"claims"`
+		Supports decimal.Decimal `json:"supports"`
+		Tips     decimal.Decimal `json:"tips"`
+	} `json:"reserved_subtotals"`
+	Total decimal.Decimal `json:"total"`
+}
+
+type Transaction struct {
+	Address            string            `json:"address"`
+	Amount             string            `json:"amount"`
+	ClaimID            string            `json:"claim_id"`
+	ClaimOp            string            `json:"claim_op"`
+	Confirmations      int               `json:"confirmations"`
+	HasSigningKey      bool              `json:"has_signing_key"`
+	Height             int               `json:"height"`
+	IsInternalTransfer bool              `json:"is_internal_transfer"`
+	IsMyInput          bool              `json:"is_my_input"`
+	IsMyOutput         bool              `json:"is_my_output"`
+	IsSpent            bool              `json:"is_spent"`
+	Name               string            `json:"name"`
+	NormalizedName     string            `json:"normalized_name"`
+	Nout               uint64            `json:"nout"`
+	PermanentUrl       string            `json:"permanent_url"`
+	SigningChannel     *Claim            `json:"signing_channel,omitempty"`
+	TimeStamp          uint64            `json:"time_stamp"`
+	Txid               string            `json:"txid"`
+	Type               string            `json:"type"`
+	Value              *lbryschema.Claim `json:"protobuf,omitempty"`
+}
+
+type TransactionSummary struct {
+	Height      int           `json:"height"`
+	Hex         string        `json:"hex"`
+	Inputs      []Transaction `json:"inputs"`
+	Outputs     []Transaction `json:"outputs"`
+	TotalFee    string        `json:"total_fee"`
+	TotalInput  string        `json:"total_input"`
+	TotalOutput string        `json:"total_output"`
+	Txid        string        `json:"txid"`
+}
+
+type AccountFundResponse TransactionSummary
+
+type Address string
+type AddressUnusedResponse Address
+type AddressListResponse struct {
+	Items []struct {
+		Account   string  `json:"account"`
+		Address   Address `json:"address"`
+		Pubkey    string  `json:"pubkey"`
+		UsedTimes uint64  `json:"used_times"`
+	} `json:"items"`
+	Page       uint64 `json:"page"`
+	PageSize   uint64 `json:"page_size"`
+	TotalPages uint64 `json:"total_pages"`
+}
+
+type ChannelExportResponse string
+type ChannelImportResponse string
+
+type ChannelListResponse struct {
+	Items      []Transaction `json:"items"`
+	Page       uint64        `json:"page"`
+	PageSize   uint64        `json:"page_size"`
+	TotalPages uint64        `json:"total_pages"`
+}
+
+type ClaimAbandonResponse struct {
+	Success bool               `json:"success"`
+	Tx      TransactionSummary `json:"tx"`
+}
+type Support struct {
+	Amount string `json:"amount"`
+	Nout   uint64 `json:"nout"`
+	Txid   string `json:"txid"`
+}
+
+type PurchaseReceipt struct {
+	Address       string `json:"file_name"`
+	Amount        string `json:"amount"`
+	ClaimID       string `json:"claim_id"`
+	Confirmations int    `json:"confirmations"`
+	Height        int    `json:"height"`
+	Nout          uint64 `json:"nout"`
+	Timestamp     uint64 `json:"timestamp"`
+	Txid          string `json:"txid"`
+	Type          string `json:"purchase"`
+}
+
+type Claim struct {
+	Address                 string           `json:"address"`
+	Amount                  string           `json:"amount"`
+	CanonicalURL            string           `json:"canonical_url"`
+	ChannelID               string           `json:"channel_id"`
+	ClaimID                 string           `json:"claim_id"`
+	ClaimOp                 string           `json:"claim_op,omitempty"`
+	Confirmations           int              `json:"confirmations"`
+	Height                  int              `json:"height"`
+	IsChange                bool             `json:"is_change,omitempty"`
+	IsChannelSignatureValid bool             `json:"is_channel_signature_valid,omitempty"`
+	IsInternalTransfer      bool             `json:"is_internal_transfer"`
+	IsMyInput               bool             `json:"is_my_input"`
+	IsMyOutput              bool             `json:"is_my_output"`
+	IsSpent                 bool             `json:"is_spent"`
+	Meta                    Meta             `json:"meta,omitempty"`
+	Name                    string           `json:"name"`
+	NormalizedName          string           `json:"normalized_name"`
+	Nout                    uint64           `json:"nout"`
+	PermanentURL            string           `json:"permanent_url"`
+	PurchaseReceipt         *PurchaseReceipt `json:"purchase_receipt,omitempty"`
+	ShortURL                string           `json:"short_url"`
+	SigningChannel          *Claim           `json:"signing_channel,omitempty"`
+	Timestamp               int              `json:"timestamp"`
+	Txid                    string           `json:"txid"`
+	Type                    string           `json:"type,omitempty"`
+	Value                   lbryschema.Claim `json:"protobuf,omitempty"`
+	ValueType               string           `json:"value_type,omitempty"`
+	AbsoluteChannelPosition int              `json:"absolute_channel_position,omitempty"`
+	ChannelName             string           `json:"channel_name,omitempty"`
+	ClaimSequence           int64            `json:"claim_sequence,omitempty"`
+	DecodedClaim            bool             `json:"decoded_claim,omitempty"`
+	EffectiveAmount         string           `json:"effective_amount,omitempty"`
+	HasSignature            *bool            `json:"has_signature,omitempty"`
+	SignatureIsValid        *bool            `json:"signature_is_valid,omitempty"`
+	Supports                []Support        `json:"supports,omitempty"`
+	ValidAtHeight           int              `json:"valid_at_height,omitempty"`
+}
+
+type Meta struct {
+	ActivationHeight  int64   `json:"activation_height,omitempty"`
+	CreationHeight    int64   `json:"creation_height,omitempty"`
+	CreationTimestamp int     `json:"creation_timestamp,omitempty"`
+	EffectiveAmount   string  `json:"effective_amount,omitempty"`
+	ExpirationHeight  int64   `json:"expiration_height,omitempty"`
+	IsControlling     bool    `json:"is_controlling,omitempty"`
+	SupportAmount     string  `json:"support_amount,omitempty"`
+	TrendingGlobal    float64 `json:"trending_global,omitempty"`
+	TrendingGroup     float64 `json:"trending_group,omitempty"`
+	TrendingLocal     float64 `json:"trending_local,omitempty"`
+	TrendingMixed     float64 `json:"trending_mixed,omitempty"`
+}
+
+const coldStorageURL = "https://s3.wasabisys.com/blobs.lbry.com/"
+
+// GetStreamSizeByMagic uses "magic" to not just estimate, but actually return the exact size of a stream
+// It does so by fetching the sd blob and the last blob from our S3 bucket, decrypting and unpadding the last blob
+// adding up all full blobs that have a known size and finally adding the real last blob size too.
+// This will only work if we host at least the sd blob and the last blob on S3, if not, this will error.
+func (c *Claim) GetStreamSizeByMagic() (streamSize uint64, e error) {
+	if c.Value.GetStream() == nil {
+		return 0, errors.Err("this claim is not a stream")
+	}
+	resp, err := http.Get(coldStorageURL + hex.EncodeToString(c.Value.GetStream().Source.SdHash))
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+	defer resp.Body.Close()
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+	sdb := &stream.SDBlob{}
+	err = sdb.UnmarshalJSON(body)
+
+	if err != nil {
+		return 0, err
+	}
+	lastBlobIndex := len(sdb.BlobInfos) - 2
+	lastBlobHash := sdb.BlobInfos[lastBlobIndex].BlobHash
+
+	if len(sdb.BlobInfos) > 2 {
+		streamSize = uint64(stream.MaxBlobSize-1) * uint64(len(sdb.BlobInfos)-2)
+	}
+
+	resp2, err := http.Get(coldStorageURL + hex.EncodeToString(lastBlobHash))
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+	defer resp2.Body.Close()
+
+	body2, err := ioutil.ReadAll(resp2.Body)
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+	defer func() {
+		if r := recover(); r != nil {
+			e = errors.Err("recovered from DecryptBlob panic for blob %s", lastBlobHash)
+		}
+	}()
+	lastBlob, err := stream.DecryptBlob(body2, sdb.Key, sdb.BlobInfos[lastBlobIndex].IV)
+	if err != nil {
+		return 0, errors.Err(err)
+	}
+
+	streamSize += uint64(len(lastBlob))
+	return streamSize, nil
+}
+
+const (
+	ProtectedContentTag      = SpecialContentType("c:members-only")
+	PurchaseContentTag       = SpecialContentType("c:purchase:")
+	RentalContentTag         = SpecialContentType("c:rental:")
+	PreorderContentTag       = SpecialContentType("c:preorder:")
+	LegacyPurchaseContentTag = SpecialContentType("purchase:")
+	LegacyRentalContentTag   = SpecialContentType("rental:")
+	LegacyPreorderContentTag = SpecialContentType("preorder:")
+	ScheduledShowContentTag  = SpecialContentType("c:scheduled:show")
+	ScheduledHideContentTag  = SpecialContentType("c:scheduled:hide")
+	UnlistedContentTag       = SpecialContentType("c:unlisted")
+)
+
+type SpecialContentType string
+
+//IsContentSpecial returns true if the claim is of a special content type
+func (c *Claim) IsContentSpecial(specialTags ...SpecialContentType) bool {
+	for _, t := range c.Value.GetTags() {
+		for _, ct := range specialTags {
+			if strings.Contains(t, string(ct)) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+type StreamListResponse struct {
+	Items      []Claim `json:"items"`
+	Page       uint64  `json:"page"`
+	PageSize   uint64  `json:"page_size"`
+	TotalPages uint64  `json:"total_pages"`
+}
+
+type ClaimListResponse struct {
+	Claims     []Claim `json:"items"`
+	Page       uint64  `json:"page"`
+	PageSize   uint64  `json:"page_size"`
+	TotalPages uint64  `json:"total_pages"`
+}
+type ClaimSearchResponse ClaimListResponse
+
+type SupportListResponse struct {
+	Items      []Claim
+	Page       uint64 `json:"page"`
+	PageSize   uint64 `json:"page_size"`
+	TotalPages uint64 `json:"total_pages"`
+}
+
+type StatusResponse struct {
+	BlobManager struct {
+		Connections struct {
+			MaxIncomingMbs   float64 `json:"max_incoming_mbs"`
+			MaxOutgoingMbs   float64 `json:"max_outgoing_mbs"`
+			TotalIncomingMbs float64 `json:"total_incoming_mbs"`
+			TotalOutgoingMbs float64 `json:"total_outgoing_mbs"`
+			TotalReceived    int64   `json:"total_received"`
+			TotalSent        int64   `json:"total_sent"`
+		} `json:"connections"`
+		FinishedBlobs int64 `json:"finished_blobs"`
+	} `json:"blob_manager"`
+	ConnectionStatus struct {
+		Code    string `json:"code"`
+		Message string `json:"message"`
+	} `json:"connection_status"`
+	Dht struct {
+		NodeID              string `json:"node_id"`
+		PeersInRoutingTable uint64 `json:"peers_in_routing_table"`
+	} `json:"dht"`
+	FfmpegStatus struct {
+		AnalyzeAudioVolume bool   `json:"analyze_audio_volume"`
+		Available          bool   `json:"available"`
+		Which              string `json:"which"`
+	} `json:"ffmpeg_status"`
+	FileManager struct {
+		ManagedFiles int64 `json:"managed_files"`
+	} `json:"file_manager"`
+	HashAnnouncer struct {
+		AnnounceQueueSize uint64 `json:"announce_queue_size"`
+	} `json:"hash_announcer"`
+	InstallationID    string   `json:"installation_id"`
+	IsRunning         bool     `json:"is_running"`
+	SkippedComponents []string `json:"skipped_components"`
+	StartupStatus     struct {
+		BlobManager          bool `json:"blob_manager"`
+		Database             bool `json:"database"`
+		Dht                  bool `json:"dht"`
+		ExchangeRateManager  bool `json:"exchange_rate_manager"`
+		FileManager          bool `json:"file_manager"`
+		HashAnnouncer        bool `json:"hash_announcer"`
+		LibtorrentComponent  bool `json:"libtorrent_component"`
+		PeerProtocolServer   bool `json:"peer_protocol_server"`
+		Upnp                 bool `json:"upnp"`
+		Wallet               bool `json:"wallet"`
+		WalletServerPayments bool `json:"wallet_server_payments"`
+	} `json:"startup_status"`
+	Upnp struct {
+		AioupnpVersion  string   `json:"aioupnp_version"`
+		DhtRedirectSet  bool     `json:"dht_redirect_set"`
+		ExternalIp      string   `json:"external_ip"`
+		Gateway         string   `json:"gateway"`
+		PeerRedirectSet bool     `json:"peer_redirect_set"`
+		Redirects       struct{} `json:"redirects"`
+	} `json:"upnp"`
+	Wallet struct {
+		AvailableServers  int    `json:"available_servers"`
+		BestBlockhash     string `json:"best_blockhash"`
+		Blocks            int    `json:"blocks"`
+		BlocksBehind      int    `json:"blocks_behind"`
+		Connected         string `json:"connected"`
+		ConnectedFeatures struct {
+			DailyFee        string `json:"daily_fee"`
+			Description     string `json:"description"`
+			DonationAddress string `json:"donation_address"`
+			GenesisHash     string `json:"genesis_hash"`
+			HashFunction    string `json:"hash_function"`
+			Hosts           struct {
+			} `json:"hosts"`
+			PaymentAddress    string      `json:"payment_address"`
+			ProtocolMax       string      `json:"protocol_max"`
+			ProtocolMin       string      `json:"protocol_min"`
+			Pruning           interface{} `json:"pruning"`
+			ServerVersion     string      `json:"server_version"`
+			TrendingAlgorithm string      `json:"trending_algorithm"`
+		} `json:"connected_features"`
+		HeadersSynchronizationProgress int `json:"headers_synchronization_progress"`
+		KnownServers                   int `json:"known_servers"`
+		Servers                        []struct {
+			Availability bool    `json:"availability"`
+			Host         string  `json:"host"`
+			Latency      float64 `json:"latency"`
+			Port         int     `json:"port"`
+		} `json:"servers"`
+	} `json:"wallet"`
+	WalletServerPayments struct {
+		MaxFee  string `json:"max_fee"`
+		Running bool   `json:"running"`
+	} `json:"wallet_server_payments"`
+}
+
+type UTXOListResponse struct {
+	Items []struct {
+		Address            string `json:"address"`
+		Amount             string `json:"amount"`
+		Confirmations      int    `json:"confirmations"`
+		Height             int    `json:"height"`
+		IsInternalTransfer bool   `json:"is_internal_transfer"`
+		IsMyInput          bool   `json:"is_my_input"`
+		IsMyOutput         bool   `json:"is_my_output"`
+		IsSpent            bool   `json:"is_spent"`
+		Nout               int    `json:"nout"`
+		Timestamp          int64  `json:"timestamp"`
+		Txid               string `json:"txid"`
+		Type               string `json:"type"`
+	} `json:"items"`
+	Page       uint64 `json:"page"`
+	PageSize   uint64 `json:"page_size"`
+	TotalPages uint64 `json:"total_pages"`
+}
+
+type UTXOReleaseResponse *string
+
+type transactionListBlob struct {
+	Address      string `json:"address"`
+	Amount       string `json:"amount"`
+	BalanceDelta string `json:"balance_delta"`
+	ClaimId      string `json:"claim_id"`
+	ClaimName    string `json:"claim_name"`
+	IsSpent      bool   `json:"is_spent"`
+	Nout         int    `json:"nout"`
+}
+
+//TODO: this repeats all the fields from transactionListBlob which doesn't make sense
+// but if i extend the type with transactionListBlob it doesn't fill the fields. does our unmarshaller crap out on these?
+type supportBlob struct {
+	Address      string `json:"address"`
+	Amount       string `json:"amount"`
+	BalanceDelta string `json:"balance_delta"`
+	ClaimId      string `json:"claim_id"`
+	ClaimName    string `json:"claim_name"`
+	IsSpent      bool   `json:"is_spent"`
+	IsTip        bool   `json:"is_tip"`
+	Nout         int    `json:"nout"`
+}
+
+type TransactionListResponse struct {
+	Items []struct {
+		AbandonInfo   []transactionListBlob `json:"abandon_info"`
+		ClaimInfo     []transactionListBlob `json:"claim_info"`
+		Confirmations int64                 `json:"confirmations"`
+		Date          string                `json:"date"`
+		Fee           string                `json:"fee"`
+		SupportInfo   []supportBlob         `json:"support_info"`
+		Timestamp     int64                 `json:"timestamp"`
+		Txid          string                `json:"txid"`
+		UpdateInfo    []transactionListBlob `json:"update_info"`
+		Value         string                `json:"value"`
+	} `json:"items"`
+	Page       uint64 `json:"page"`
+	PageSize   uint64 `json:"page_size"`
+	TotalPages uint64 `json:"total_pages"`
+}
+
+type VersionResponse struct {
+	Build   string `json:"build"`
+	Desktop string `json:"desktop"`
+	Distro  struct {
+		Codename     string `json:"codename"`
+		ID           string `json:"id"`
+		Like         string `json:"like"`
+		Version      string `json:"version"`
+		VersionParts struct {
+			BuildNumber string `json:"build_number"`
+			Major       string `json:"major"`
+			Minor       string `json:"minor"`
+		} `json:"version_parts"`
+	} `json:"distro"`
+	LbrynetVersion string `json:"lbrynet_version"`
+	OsRelease      string `json:"os_release"`
+	OsSystem       string `json:"os_system"`
+	Platform       string `json:"platform"`
+	Processor      string `json:"processor"`
+	PythonVersion  string `json:"python_version"`
+	Version        string `json:"version"`
+}
+
+type ResolveResponse map[string]Claim
+
+type ClaimShowResponse *Claim
+
+type Wallet struct {
+	ID   string `json:"id"`
+	Name string `json:"name"`
+}
+
+type WalletList struct {
+	Items      []Wallet `json:"items"`
+	Page       uint64   `json:"page"`
+	PageSize   uint64   `json:"page_size"`
+	TotalPages uint64   `json:"total_pages"`
+}

extras/lbryinc/client.go

@@ -0,0 +1,269 @@
+package lbryinc
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+
+	"golang.org/x/oauth2"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	defaultServerAddress = "https://api.odysee.tv"
+	timeout              = 5 * time.Second
+	headerForwardedFor   = "X-Forwarded-For"
+
+	userObjectPath             = "user"
+	userMeMethod               = "me"
+	userHasVerifiedEmailMethod = "has_verified_email"
+)
+
+// Client stores data about internal-apis call it is about to make.
+type Client struct {
+	AuthToken     string
+	OAuthToken    oauth2.TokenSource
+	Logger        *log.Logger
+	serverAddress string
+	extraHeaders  map[string]string
+}
+
+// ClientOpts allow to provide extra parameters to NewClient:
+// - ServerAddress
+// - RemoteIP — to forward the IP of a frontend client making the request
+type ClientOpts struct {
+	ServerAddress string
+	RemoteIP      string
+}
+
+// APIResponse reflects internal-apis JSON response format.
+type APIResponse struct {
+	Success bool        `json:"success"`
+	Error   *string     `json:"error"`
+	Data    interface{} `json:"data"`
+}
+
+type data struct {
+	obj   map[string]interface{}
+	array []interface{}
+}
+
+func (d data) IsObject() bool {
+	return d.obj != nil
+}
+
+func (d data) IsArray() bool {
+	return d.array != nil
+}
+
+func (d data) Object() (map[string]interface{}, error) {
+	if d.obj == nil {
+		return nil, errors.New("no object data found")
+	}
+	return d.obj, nil
+}
+
+func (d data) Array() ([]interface{}, error) {
+	if d.array == nil {
+		return nil, errors.New("no array data found")
+	}
+	return d.array, nil
+}
+
+// APIError wraps errors returned by LBRY API server to discern them from other kinds (like http errors).
+type APIError struct {
+	Err error
+}
+
+func (e APIError) Error() string {
+	return fmt.Sprintf("api error: %v", e.Err)
+}
+
+// ResponseData is a map containing parsed json response.
+type ResponseData interface {
+	IsObject() bool
+	IsArray() bool
+	Object() (map[string]interface{}, error)
+	Array() ([]interface{}, error)
+}
+
+func makeMethodPath(obj, method string) string {
+	return fmt.Sprintf("/%s/%s", obj, method)
+}
+
+// NewClient returns a client instance for internal-apis. It requires authToken to be provided
+// for authentication.
+func NewClient(authToken string, opts *ClientOpts) Client {
+	c := Client{
+		serverAddress: defaultServerAddress,
+		extraHeaders:  make(map[string]string),
+		AuthToken:     authToken,
+		Logger:        log.StandardLogger(),
+	}
+	if opts != nil {
+		if opts.ServerAddress != "" {
+			c.serverAddress = opts.ServerAddress
+		}
+		if opts.RemoteIP != "" {
+			c.extraHeaders[headerForwardedFor] = opts.RemoteIP
+		}
+	}
+
+	return c
+}
+
+// NewOauthClient returns a client instance for internal-apis. It requires Oauth Token Source to be provided
+// for authentication.
+func NewOauthClient(token oauth2.TokenSource, opts *ClientOpts) Client {
+	c := Client{
+		serverAddress: defaultServerAddress,
+		extraHeaders:  make(map[string]string),
+		OAuthToken:    token,
+		Logger:        log.StandardLogger(),
+	}
+	if opts != nil {
+		if opts.ServerAddress != "" {
+			c.serverAddress = opts.ServerAddress
+		}
+		if opts.RemoteIP != "" {
+			c.extraHeaders[headerForwardedFor] = opts.RemoteIP
+		}
+	}
+
+	return c
+}
+
+func (c Client) getEndpointURL(object, method string) string {
+	return fmt.Sprintf("%s%s", c.serverAddress, makeMethodPath(object, method))
+}
+
+func (c Client) getEndpointURLFromPath(path string) string {
+	return fmt.Sprintf("%s%s", c.serverAddress, path)
+}
+
+func (c Client) prepareParams(params map[string]interface{}) (string, error) {
+	form := url.Values{}
+	if c.AuthToken != "" {
+		form.Add("auth_token", c.AuthToken)
+	} else if c.OAuthToken == nil {
+		return "", errors.New("oauth token source must be supplied")
+	}
+	for k, v := range params {
+		if k == "auth_token" {
+			return "", errors.New("extra auth_token supplied in request params")
+		}
+		form.Add(k, fmt.Sprintf("%v", v))
+	}
+	return form.Encode(), nil
+}
+
+func (c Client) doCall(url string, payload string) ([]byte, error) {
+	var body []byte
+	c.Logger.Debugf("sending payload: %s", payload)
+	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer([]byte(payload)))
+	if err != nil {
+		return body, err
+	}
+
+	req.Header.Add("Accept", "application/json")
+	req.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	if c.OAuthToken != nil {
+		t, err := c.OAuthToken.Token()
+		if err != nil {
+			return nil, err
+		}
+		if t.Type() != "Bearer" {
+			return nil, errors.New("internal-apis requires an oAuth token of type 'Bearer'")
+		}
+		t.SetAuthHeader(req)
+	}
+
+	for k, v := range c.extraHeaders {
+		req.Header.Set(k, v)
+	}
+
+	client := &http.Client{Timeout: timeout}
+	r, err := client.Do(req)
+	if err != nil {
+		return body, err
+	}
+	if r.StatusCode >= 500 {
+		return body, fmt.Errorf("server returned non-OK status: %v", r.StatusCode)
+	}
+	defer r.Body.Close()
+	return ioutil.ReadAll(r.Body)
+}
+
+// CallResource calls a remote internal-apis server resource, returning a response,
+// wrapped into standardized API Response struct.
+func (c Client) CallResource(object, method string, params map[string]interface{}) (ResponseData, error) {
+	var d data
+	payload, err := c.prepareParams(params)
+	if err != nil {
+		return d, err
+	}
+
+	body, err := c.doCall(c.getEndpointURL(object, method), payload)
+	if err != nil {
+		return d, err
+	}
+	var ar APIResponse
+	err = json.Unmarshal(body, &ar)
+	if err != nil {
+		return d, err
+	}
+	if !ar.Success {
+		return d, APIError{errors.New(*ar.Error)}
+	}
+	if v, ok := ar.Data.([]interface{}); ok {
+		d.array = v
+	} else if v, ok := ar.Data.(map[string]interface{}); ok {
+		d.obj = v
+	}
+	return d, err
+}
+
+// Call calls a remote internal-apis server, returning a response,
+// wrapped into standardized API Response struct.
+func (c Client) Call(path string, params map[string]interface{}) (ResponseData, error) {
+	var d data
+	payload, err := c.prepareParams(params)
+	if err != nil {
+		return d, err
+	}
+
+	body, err := c.doCall(c.getEndpointURLFromPath(path), payload)
+	if err != nil {
+		return d, err
+	}
+	var ar APIResponse
+	err = json.Unmarshal(body, &ar)
+	if err != nil {
+		return d, err
+	}
+	if !ar.Success {
+		return d, APIError{errors.New(*ar.Error)}
+	}
+	if v, ok := ar.Data.([]interface{}); ok {
+		d.array = v
+	} else if v, ok := ar.Data.(map[string]interface{}); ok {
+		d.obj = v
+	}
+	return d, err
+}
+
+// UserMe returns user details for the user associated with the current auth_token.
+func (c Client) UserMe() (ResponseData, error) {
+	return c.CallResource(userObjectPath, userMeMethod, map[string]interface{}{})
+}
+
+// UserHasVerifiedEmail calls has_verified_email method.
+func (c Client) UserHasVerifiedEmail() (ResponseData, error) {
+	return c.CallResource(userObjectPath, userHasVerifiedEmailMethod, map[string]interface{}{})
+}

extras/lbryinc/client_test.go

@@ -0,0 +1,182 @@
+package lbryinc
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+
+	"golang.org/x/oauth2"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func launchDummyServer(lastReq **http.Request, path, response string, status int) *httptest.Server {
+	return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if lastReq != nil {
+			*lastReq = &*r
+		}
+		authT := r.FormValue("auth_token")
+		if authT == "" {
+			accessT := r.Header.Get("Authorization")
+			if accessT == "" {
+				w.WriteHeader(http.StatusUnauthorized)
+				return
+			}
+		}
+		if r.URL.Path != path {
+			fmt.Printf("path doesn't match: %v != %v", r.URL.Path, path)
+			w.WriteHeader(http.StatusNotFound)
+		} else {
+			w.Header().Set("Content-Type", "application/json; charset=utf-8")
+			w.WriteHeader(status)
+			w.Write([]byte(response))
+		}
+	}))
+}
+
+func TestUserMe(t *testing.T) {
+	ts := launchDummyServer(nil, makeMethodPath(userObjectPath, userMeMethod), userMeResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL})
+	r, err := c.UserMe()
+	assert.Nil(t, err)
+	robj, err := r.Object()
+	if err != nil {
+		t.Fatal(err)
+	}
+	assert.Equal(t, "user@lbry.tv", robj["primary_email"])
+}
+
+func TestListFiltered(t *testing.T) {
+	ts := launchDummyServer(nil, "/file/list_filtered", listFilteredResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL})
+	r, err := c.CallResource("file", "list_filtered", map[string]interface{}{"with_claim_id": "true"})
+	assert.Nil(t, err)
+	assert.True(t, r.IsArray())
+	_, err = r.Array()
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestUserHasVerifiedEmail(t *testing.T) {
+	ts := launchDummyServer(nil, makeMethodPath(userObjectPath, userHasVerifiedEmailMethod), userHasVerifiedEmailResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL})
+	r, err := c.UserHasVerifiedEmail()
+	assert.Nil(t, err)
+	robj, err := r.Object()
+	if err != nil {
+		t.Error(err)
+	}
+	assert.EqualValues(t, 12345, robj["user_id"])
+	assert.Equal(t, true, robj["has_verified_email"])
+}
+
+func TestUserHasVerifiedEmailOAuth(t *testing.T) {
+	ts := launchDummyServer(nil, makeMethodPath(userObjectPath, userHasVerifiedEmailMethod), userHasVerifiedEmailResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewOauthClient(oauth2.StaticTokenSource(&oauth2.Token{AccessToken: "Test-Access-Token"}), &ClientOpts{ServerAddress: ts.URL})
+	r, err := c.UserHasVerifiedEmail()
+	assert.Nil(t, err)
+	robj, err := r.Object()
+	if err != nil {
+		t.Error(err)
+	}
+	assert.EqualValues(t, 12345, robj["user_id"])
+	assert.Equal(t, true, robj["has_verified_email"])
+}
+
+func TestRemoteIP(t *testing.T) {
+	var req *http.Request
+	ts := launchDummyServer(&req, makeMethodPath(userObjectPath, userMeMethod), userMeResponse, http.StatusOK)
+	defer ts.Close()
+
+	c := NewClient("realToken", &ClientOpts{ServerAddress: ts.URL, RemoteIP: "8.8.8.8"})
+	_, err := c.UserMe()
+	assert.Nil(t, err)
+	assert.Equal(t, []string{"8.8.8.8"}, req.Header["X-Forwarded-For"])
+}
+
+func TestWrongToken(t *testing.T) {
+	c := NewClient("zcasdasc", nil)
+
+	r, err := c.UserHasVerifiedEmail()
+	assert.False(t, r.IsObject())
+	assert.EqualError(t, err, "api error: could not authenticate user")
+	assert.ErrorAs(t, err, &APIError{})
+}
+
+func TestHTTPError(t *testing.T) {
+	c := NewClient("zcasdasc", &ClientOpts{ServerAddress: "http://lolcathost"})
+
+	r, err := c.UserHasVerifiedEmail()
+	assert.False(t, r.IsObject())
+	assert.EqualError(t, err, `Post "http://lolcathost/user/has_verified_email": dial tcp: lookup lolcathost: no such host`)
+}
+
+func TestGatewayError(t *testing.T) {
+	var req *http.Request
+	ts := launchDummyServer(&req, makeMethodPath(userObjectPath, userHasVerifiedEmailMethod), "", http.StatusBadGateway)
+	defer ts.Close()
+	c := NewClient("zcasdasc", &ClientOpts{ServerAddress: ts.URL})
+
+	r, err := c.UserHasVerifiedEmail()
+	assert.False(t, r.IsObject())
+	assert.EqualError(t, err, `server returned non-OK status: 502`)
+}
+
+const userMeResponse = `{
+	"success": true,
+	"error": null,
+	"data": {
+		"id": 12345,
+		"language": "en",
+		"given_name": null,
+		"family_name": null,
+		"created_at": "2019-01-17T12:13:06Z",
+		"updated_at": "2019-05-02T13:57:59Z",
+		"invited_by_id": null,
+		"invited_at": null,
+		"invites_remaining": 0,
+		"invite_reward_claimed": false,
+		"is_email_enabled": true,
+		"manual_approval_user_id": 654,
+		"reward_status_change_trigger": "manual",
+		"primary_email": "user@lbry.tv",
+		"has_verified_email": true,
+		"is_identity_verified": false,
+		"is_reward_approved": true,
+		"groups": []
+	}
+}`
+
+const userHasVerifiedEmailResponse = `{
+	"success": true,
+	"error": null,
+	"data": {
+	  "user_id": 12345,
+	  "has_verified_email": true
+	}
+}`
+
+const listFilteredResponse = `{
+  "success": true,
+  "error": null,
+  "data": [
+    {
+      "claim_id": "322ce77e9085d9da42279c790f7c9755b4916fca",
+      "outpoint": "20e04af21a569061ced7aa1801a43b4ed4839dfeb79919ea49a4059c7fe114c5:0"
+    },
+    {
+      "claim_id": "61496c567badcd98b82d9a700a8d56fd8a5fa8fb",
+      "outpoint": "657e4ec774524b326f9d3ecb9f468ea085bd1f3d450565f0330feca02e8fd25b:0"
+    }
+  ]
+}`

extras/null/LICENSE

@@ -0,0 +1,11 @@
+Copyright for portions of project null-extended are held by *Greg Roseberry, 2014* as part of project null.
+All other copyright for project null-extended are held by *Patrick O'Brien, 2016*.
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

extras/null/README.md

@@ -0,0 +1,89 @@
+## null-extended [![GoDoc](https://godoc.org/github.com/nullbio/null?status.svg)](https://godoc.org/github.com/nullbio/null) [![Coverage](http://gocover.io/_badge/github.com/nullbio/null)](http://gocover.io/github.com/nullbio/null)
+
+*Forked from https://github.com/nullbio/null*
+
+null-extended is a library with reasonable options for dealing with nullable SQL and JSON values
+
+Types in `null` will only be considered null on null input, and will JSON encode to `null`.
+
+All types implement `sql.Scanner` and `driver.Valuer`, so you can use this library in place of `sql.NullXXX`. All types also implement: `encoding.TextMarshaler`, `encoding.TextUnmarshaler`, `json.Marshaler`, `json.Unmarshaler` and `sql.Scanner`.
+
+---
+
+Install:
+
+`go get -u "gopkg.in/nullbio/null.v6"`
+
+### null package
+
+`import "gopkg.in/nullbio/null.v6"`
+
+The following are all types supported in this package. All types will marshal to JSON null if Invalid or SQL source data is null.
+
+#### null.JSON
+Nullable []byte.
+
+Will marshal to JSON null if Invalid. []byte{} input will not produce an Invalid JSON, but []byte(nil) will. This should be used for storing raw JSON in the database.
+
+Also has `null.JSON.Marshal` and `null.JSON.Unmarshal` helpers to marshal and unmarshal foreign objects.
+
+#### null.Bytes
+Nullable []byte.
+
+[]byte{} input will not produce an Invalid Bytes, but []byte(nil) will. This should be used for storing binary data (bytea in PSQL for example) in the database.
+
+#### null.String
+Nullable string.
+
+#### null.Byte
+Nullable byte.
+
+#### null.Bool
+Nullable bool.
+
+#### null.Time
+Nullable time.Time
+
+Marshals to JSON null if SQL source data is null. Uses `time.Time`'s marshaler.
+
+#### null.Float32
+Nullable float32.
+
+#### null.Float64
+Nullable float64.
+
+#### null.Int
+Nullable int.
+
+#### null.Int8
+Nullable int8.
+
+#### null.Int16
+Nullable int16.
+
+#### null.Int32
+Nullable int32.
+
+#### null.Int64
+Nullable int64.
+
+#### null.Uint
+Nullable uint.
+
+#### null.Uint8
+Nullable uint8.
+
+#### null.Uint16
+Nullable uint16.
+
+#### null.Uint32
+Nullable int32.
+
+#### null.Int64
+Nullable uint64.
+
+### Bugs
+`json`'s `",omitempty"` struct tag does not work correctly right now. It will never omit a null or empty String. This might be [fixed eventually](https://github.com/golang/go/issues/4357).
+
+### License
+BSD

extras/null/bool.go

@@ -0,0 +1,133 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Bool is a nullable bool.
+type Bool struct {
+	Bool  bool
+	Valid bool
+}
+
+// NewBool creates a new Bool
+func NewBool(b bool, valid bool) Bool {
+	return Bool{
+		Bool:  b,
+		Valid: valid,
+	}
+}
+
+// BoolFrom creates a new Bool that will always be valid.
+func BoolFrom(b bool) Bool {
+	return NewBool(b, true)
+}
+
+// BoolFromPtr creates a new Bool that will be null if f is nil.
+func BoolFromPtr(b *bool) Bool {
+	if b == nil {
+		return NewBool(false, false)
+	}
+	return NewBool(*b, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (b *Bool) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		b.Bool = false
+		b.Valid = false
+		return nil
+	}
+
+	if err := json.Unmarshal(data, &b.Bool); err != nil {
+		return err
+	}
+
+	b.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (b *Bool) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		b.Valid = false
+		return nil
+	}
+
+	str := string(text)
+	switch str {
+	case "true":
+		b.Bool = true
+	case "false":
+		b.Bool = false
+	default:
+		b.Valid = false
+		return errors.New("invalid input:" + str)
+	}
+	b.Valid = true
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler.
+func (b Bool) MarshalJSON() ([]byte, error) {
+	if !b.Valid {
+		return NullBytes, nil
+	}
+	if !b.Bool {
+		return []byte("false"), nil
+	}
+	return []byte("true"), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (b Bool) MarshalText() ([]byte, error) {
+	if !b.Valid {
+		return []byte{}, nil
+	}
+	if !b.Bool {
+		return []byte("false"), nil
+	}
+	return []byte("true"), nil
+}
+
+// SetValid changes this Bool's value and also sets it to be non-null.
+func (b *Bool) SetValid(v bool) {
+	b.Bool = v
+	b.Valid = true
+}
+
+// Ptr returns a pointer to this Bool's value, or a nil pointer if this Bool is null.
+func (b Bool) Ptr() *bool {
+	if !b.Valid {
+		return nil
+	}
+	return &b.Bool
+}
+
+// IsNull returns true for invalid Bools, for future omitempty support (Go 1.4?)
+func (b Bool) IsNull() bool {
+	return !b.Valid
+}
+
+// Scan implements the Scanner interface.
+func (b *Bool) Scan(value interface{}) error {
+	if value == nil {
+		b.Bool, b.Valid = false, false
+		return nil
+	}
+	b.Valid = true
+	return convert.ConvertAssign(&b.Bool, value)
+}
+
+// Value implements the driver Valuer interface.
+func (b Bool) Value() (driver.Value, error) {
+	if !b.Valid {
+		return nil, nil
+	}
+	return b.Bool, nil
+}

extras/null/bool_test.go

@@ -0,0 +1,196 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	boolJSON  = []byte(`true`)
+	falseJSON = []byte(`false`)
+)
+
+func TestBoolFrom(t *testing.T) {
+	b := BoolFrom(true)
+	assertBool(t, b, "BoolFrom()")
+
+	zero := BoolFrom(false)
+	if !zero.Valid {
+		t.Error("BoolFrom(false)", "is invalid, but should be valid")
+	}
+}
+
+func TestBoolFromPtr(t *testing.T) {
+	n := true
+	bptr := &n
+	b := BoolFromPtr(bptr)
+	assertBool(t, b, "BoolFromPtr()")
+
+	null := BoolFromPtr(nil)
+	assertNullBool(t, null, "BoolFromPtr(nil)")
+}
+
+func TestUnmarshalBool(t *testing.T) {
+	var b Bool
+	err := json.Unmarshal(boolJSON, &b)
+	maybePanic(err)
+	assertBool(t, b, "bool json")
+
+	var null Bool
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullBool(t, null, "null json")
+
+	var badType Bool
+	err = json.Unmarshal(intJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullBool(t, badType, "wrong type json")
+
+	var invalid Bool
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+}
+
+func TestTextUnmarshalBool(t *testing.T) {
+	var b Bool
+	err := b.UnmarshalText([]byte("true"))
+	maybePanic(err)
+	assertBool(t, b, "UnmarshalText() bool")
+
+	var zero Bool
+	err = zero.UnmarshalText([]byte("false"))
+	maybePanic(err)
+	assertFalseBool(t, zero, "UnmarshalText() false")
+
+	var blank Bool
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullBool(t, blank, "UnmarshalText() empty bool")
+
+	var invalid Bool
+	err = invalid.UnmarshalText([]byte(":D"))
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullBool(t, invalid, "invalid json")
+}
+
+func TestMarshalBool(t *testing.T) {
+	b := BoolFrom(true)
+	data, err := json.Marshal(b)
+	maybePanic(err)
+	assertJSONEquals(t, data, "true", "non-empty json marshal")
+
+	zero := NewBool(false, true)
+	data, err = json.Marshal(zero)
+	maybePanic(err)
+	assertJSONEquals(t, data, "false", "zero json marshal")
+
+	// invalid values should be encoded as null
+	null := NewBool(false, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalBoolText(t *testing.T) {
+	b := BoolFrom(true)
+	data, err := b.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "true", "non-empty text marshal")
+
+	zero := NewBool(false, true)
+	data, err = zero.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "false", "zero text marshal")
+
+	// invalid values should be encoded as null
+	null := NewBool(false, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestBoolPointer(t *testing.T) {
+	b := BoolFrom(true)
+	ptr := b.Ptr()
+	if *ptr != true {
+		t.Errorf("bad %s bool: %#v ≠ %v\n", "pointer", ptr, true)
+	}
+
+	null := NewBool(false, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s bool: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestBoolIsNull(t *testing.T) {
+	b := BoolFrom(true)
+	if b.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewBool(false, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewBool(false, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestBoolSetValid(t *testing.T) {
+	change := NewBool(false, false)
+	assertNullBool(t, change, "SetValid()")
+	change.SetValid(true)
+	assertBool(t, change, "SetValid()")
+}
+
+func TestBoolScan(t *testing.T) {
+	var b Bool
+	err := b.Scan(true)
+	maybePanic(err)
+	assertBool(t, b, "scanned bool")
+
+	var null Bool
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullBool(t, null, "scanned null")
+}
+
+func assertBool(t *testing.T, b Bool, from string) {
+	if b.Bool != true {
+		t.Errorf("bad %s bool: %v ≠ %v\n", from, b.Bool, true)
+	}
+	if !b.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertFalseBool(t *testing.T, b Bool, from string) {
+	if b.Bool != false {
+		t.Errorf("bad %s bool: %v ≠ %v\n", from, b.Bool, false)
+	}
+	if !b.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullBool(t *testing.T, b Bool, from string) {
+	if b.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/byte.go

@@ -0,0 +1,135 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+)
+
+// Byte is an nullable int.
+type Byte struct {
+	Byte  byte
+	Valid bool
+}
+
+// NewByte creates a new Byte
+func NewByte(b byte, valid bool) Byte {
+	return Byte{
+		Byte:  b,
+		Valid: valid,
+	}
+}
+
+// ByteFrom creates a new Byte that will always be valid.
+func ByteFrom(b byte) Byte {
+	return NewByte(b, true)
+}
+
+// ByteFromPtr creates a new Byte that be null if i is nil.
+func ByteFromPtr(b *byte) Byte {
+	if b == nil {
+		return NewByte(0, false)
+	}
+	return NewByte(*b, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (b *Byte) UnmarshalJSON(data []byte) error {
+	if len(data) == 0 || bytes.Equal(data, NullBytes) {
+		b.Valid = false
+		b.Byte = 0
+		return nil
+	}
+
+	var x string
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if len(x) > 1 {
+		return errors.New("json: cannot convert to byte, text len is greater than one")
+	}
+
+	b.Byte = x[0]
+	b.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (b *Byte) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		b.Valid = false
+		return nil
+	}
+
+	if len(text) > 1 {
+		return errors.New("text: cannot convert to byte, text len is greater than one")
+	}
+
+	b.Valid = true
+	b.Byte = text[0]
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler.
+func (b Byte) MarshalJSON() ([]byte, error) {
+	if !b.Valid {
+		return NullBytes, nil
+	}
+	return []byte{'"', b.Byte, '"'}, nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (b Byte) MarshalText() ([]byte, error) {
+	if !b.Valid {
+		return []byte{}, nil
+	}
+	return []byte{b.Byte}, nil
+}
+
+// SetValid changes this Byte's value and also sets it to be non-null.
+func (b *Byte) SetValid(n byte) {
+	b.Byte = n
+	b.Valid = true
+}
+
+// Ptr returns a pointer to this Byte's value, or a nil pointer if this Byte is null.
+func (b Byte) Ptr() *byte {
+	if !b.Valid {
+		return nil
+	}
+	return &b.Byte
+}
+
+// IsNull returns true for invalid Bytes, for future omitempty support (Go 1.4?)
+func (b Byte) IsNull() bool {
+	return !b.Valid
+}
+
+// Scan implements the Scanner interface.
+func (b *Byte) Scan(value interface{}) error {
+	if value == nil {
+		b.Byte, b.Valid = 0, false
+		return nil
+	}
+
+	val := value.(string)
+	if len(val) == 0 {
+		b.Valid = false
+		b.Byte = 0
+		return nil
+	}
+
+	b.Valid = true
+	b.Byte = byte(val[0])
+	return nil
+}
+
+// Value implements the driver Valuer interface.
+func (b Byte) Value() (driver.Value, error) {
+	if !b.Valid {
+		return nil, nil
+	}
+	return []byte{b.Byte}, nil
+}

extras/null/byte_test.go

@@ -0,0 +1,168 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	byteJSON = []byte(`"b"`)
+)
+
+func TestByteFrom(t *testing.T) {
+	i := ByteFrom('b')
+	assertByte(t, i, "ByteFrom()")
+
+	zero := ByteFrom(0)
+	if !zero.Valid {
+		t.Error("ByteFrom(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestByteFromPtr(t *testing.T) {
+	n := byte('b')
+	iptr := &n
+	i := ByteFromPtr(iptr)
+	assertByte(t, i, "ByteFromPtr()")
+
+	null := ByteFromPtr(nil)
+	assertNullByte(t, null, "ByteFromPtr(nil)")
+}
+
+func TestUnmarshalByte(t *testing.T) {
+	var null Byte
+	err := json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullByte(t, null, "null json")
+
+	var badType Byte
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullByte(t, badType, "wrong type json")
+
+	var invalid Byte
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullByte(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonByteegerNumber(t *testing.T) {
+	var i Byte
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to int")
+	}
+}
+
+func TestTextUnmarshalByte(t *testing.T) {
+	var i Byte
+	err := i.UnmarshalText([]byte("b"))
+	maybePanic(err)
+	assertByte(t, i, "UnmarshalText() int")
+
+	var blank Byte
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullByte(t, blank, "UnmarshalText() empty int")
+}
+
+func TestMarshalByte(t *testing.T) {
+	i := ByteFrom('b')
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, `"b"`, "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewByte(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalByteText(t *testing.T) {
+	i := ByteFrom('b')
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "b", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewByte(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestBytePointer(t *testing.T) {
+	i := ByteFrom('b')
+	ptr := i.Ptr()
+	if *ptr != 'b' {
+		t.Errorf("bad %s int: %#v ≠ %d\n", "pointer", ptr, 'b')
+	}
+
+	null := NewByte(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s int: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestByteIsNull(t *testing.T) {
+	i := ByteFrom('b')
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewByte(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewByte(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestByteSetValid(t *testing.T) {
+	change := NewByte(0, false)
+	assertNullByte(t, change, "SetValid()")
+	change.SetValid('b')
+	assertByte(t, change, "SetValid()")
+}
+
+func TestByteScan(t *testing.T) {
+	var i Byte
+	err := i.Scan("b")
+	maybePanic(err)
+	assertByte(t, i, "scanned int")
+
+	var null Byte
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullByte(t, null, "scanned null")
+}
+
+func assertByte(t *testing.T, i Byte, from string) {
+	if i.Byte != 'b' {
+		t.Errorf("bad %s int: %d ≠ %d\n", from, i.Byte, 'b')
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullByte(t *testing.T, i Byte, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/bytes.go

@@ -0,0 +1,124 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// NullBytes is a global byte slice of JSON null
+var NullBytes = []byte("null")
+
+// Bytes is a nullable []byte.
+type Bytes struct {
+	Bytes []byte
+	Valid bool
+}
+
+// NewBytes creates a new Bytes
+func NewBytes(b []byte, valid bool) Bytes {
+	return Bytes{
+		Bytes: b,
+		Valid: valid,
+	}
+}
+
+// BytesFrom creates a new Bytes that will be invalid if nil.
+func BytesFrom(b []byte) Bytes {
+	return NewBytes(b, b != nil)
+}
+
+// BytesFromPtr creates a new Bytes that will be invalid if nil.
+func BytesFromPtr(b *[]byte) Bytes {
+	if b == nil {
+		return NewBytes(nil, false)
+	}
+	n := NewBytes(*b, true)
+	return n
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (b *Bytes) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		b.Valid = false
+		b.Bytes = nil
+		return nil
+	}
+
+	var s string
+	if err := json.Unmarshal(data, &s); err != nil {
+		return err
+	}
+
+	b.Bytes = []byte(s)
+	b.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (b *Bytes) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		b.Bytes = nil
+		b.Valid = false
+	} else {
+		b.Bytes = append(b.Bytes[0:0], text...)
+		b.Valid = true
+	}
+
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler.
+func (b Bytes) MarshalJSON() ([]byte, error) {
+	if len(b.Bytes) == 0 || b.Bytes == nil {
+		return NullBytes, nil
+	}
+	return b.Bytes, nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (b Bytes) MarshalText() ([]byte, error) {
+	if !b.Valid {
+		return nil, nil
+	}
+	return b.Bytes, nil
+}
+
+// SetValid changes this Bytes's value and also sets it to be non-null.
+func (b *Bytes) SetValid(n []byte) {
+	b.Bytes = n
+	b.Valid = true
+}
+
+// Ptr returns a pointer to this Bytes's value, or a nil pointer if this Bytes is null.
+func (b Bytes) Ptr() *[]byte {
+	if !b.Valid {
+		return nil
+	}
+	return &b.Bytes
+}
+
+// IsNull returns true for null or zero Bytes's, for future omitempty support (Go 1.4?)
+func (b Bytes) IsNull() bool {
+	return !b.Valid
+}
+
+// Scan implements the Scanner interface.
+func (b *Bytes) Scan(value interface{}) error {
+	if value == nil {
+		b.Bytes, b.Valid = []byte{}, false
+		return nil
+	}
+	b.Valid = true
+	return convert.ConvertAssign(&b.Bytes, value)
+}
+
+// Value implements the driver Valuer interface.
+func (b Bytes) Value() (driver.Value, error) {
+	if !b.Valid {
+		return nil, nil
+	}
+	return b.Bytes, nil
+}

extras/null/bytes_test.go

@@ -0,0 +1,167 @@
+package null
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+)
+
+var (
+	bytesJSON = []byte(`"hello"`)
+)
+
+func TestBytesFrom(t *testing.T) {
+	i := BytesFrom([]byte(`hello`))
+	assertBytes(t, i, "BytesFrom()")
+
+	zero := BytesFrom(nil)
+	if zero.Valid {
+		t.Error("BytesFrom(nil)", "is valid, but should be invalid")
+	}
+
+	zero = BytesFrom([]byte{})
+	if !zero.Valid {
+		t.Error("BytesFrom([]byte{})", "is invalid, but should be valid")
+	}
+}
+
+func TestBytesFromPtr(t *testing.T) {
+	n := []byte(`hello`)
+	iptr := &n
+	i := BytesFromPtr(iptr)
+	assertBytes(t, i, "BytesFromPtr()")
+
+	null := BytesFromPtr(nil)
+	assertNullBytes(t, null, "BytesFromPtr(nil)")
+}
+
+func TestUnmarshalBytes(t *testing.T) {
+	var i Bytes
+	err := json.Unmarshal(bytesJSON, &i)
+	maybePanic(err)
+	assertBytes(t, i, "[]byte json")
+
+	var ni Bytes
+	err = ni.UnmarshalJSON([]byte{})
+	if err == nil {
+		t.Errorf("Expected error")
+	}
+
+	var null Bytes
+	err = null.UnmarshalJSON([]byte("null"))
+	if null.Valid == true {
+		t.Errorf("expected Valid to be false, got true")
+	}
+	if null.Bytes != nil {
+		t.Errorf("Expected Bytes to be nil, but was not: %#v %#v", null.Bytes, []byte(`null`))
+	}
+}
+
+func TestTextUnmarshalBytes(t *testing.T) {
+	var i Bytes
+	err := i.UnmarshalText([]byte(`hello`))
+	maybePanic(err)
+	assertBytes(t, i, "UnmarshalText() []byte")
+
+	var blank Bytes
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullBytes(t, blank, "UnmarshalText() empty []byte")
+}
+
+func TestMarshalBytes(t *testing.T) {
+	i := BytesFrom([]byte(`"hello"`))
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, `"hello"`, "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewBytes(nil, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalBytesText(t *testing.T) {
+	i := BytesFrom([]byte(`"hello"`))
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, `"hello"`, "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewBytes(nil, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestBytesPointer(t *testing.T) {
+	i := BytesFrom([]byte(`"hello"`))
+	ptr := i.Ptr()
+	if !bytes.Equal(*ptr, []byte(`"hello"`)) {
+		t.Errorf("bad %s []byte: %#v ≠ %s\n", "pointer", ptr, `"hello"`)
+	}
+
+	null := NewBytes(nil, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s []byte: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestBytesIsNull(t *testing.T) {
+	i := BytesFrom([]byte(`"hello"`))
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewBytes(nil, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewBytes(nil, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestBytesSetValid(t *testing.T) {
+	change := NewBytes(nil, false)
+	assertNullBytes(t, change, "SetValid()")
+	change.SetValid([]byte(`hello`))
+	assertBytes(t, change, "SetValid()")
+}
+
+func TestBytesScan(t *testing.T) {
+	var i Bytes
+	err := i.Scan(`hello`)
+	maybePanic(err)
+	assertBytes(t, i, "Scan() []byte")
+
+	var null Bytes
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullBytes(t, null, "scanned null")
+}
+
+func assertBytes(t *testing.T, i Bytes, from string) {
+	if !bytes.Equal(i.Bytes, []byte("hello")) {
+		t.Errorf("bad %s []byte: %v ≠ %v\n", from, string(i.Bytes), string([]byte(`hello`)))
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullBytes(t *testing.T, i Bytes, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/convert/convert.go

@@ -0,0 +1,266 @@
+package convert
+
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Type conversions for Scan.
+// These functions are copied from database/sql/convert.go build 1.6.2
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"errors"
+	"fmt"
+	"reflect"
+	"strconv"
+	"time"
+)
+
+var errNilPtr = errors.New("destination pointer is nil") // embedded in descriptive error
+
+// ConvertAssign copies to dest the value in src, converting it if possible.
+// An error is returned if the copy would result in loss of information.
+// dest should be a pointer type.
+func ConvertAssign(dest, src interface{}) error {
+	// Common cases, without reflect.
+	switch s := src.(type) {
+	case string:
+		switch d := dest.(type) {
+		case *string:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = s
+			return nil
+		case *[]byte:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = []byte(s)
+			return nil
+		}
+	case []byte:
+		switch d := dest.(type) {
+		case *string:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = string(s)
+			return nil
+		case *interface{}:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = cloneBytes(s)
+			return nil
+		case *[]byte:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = cloneBytes(s)
+			return nil
+		case *sql.RawBytes:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = s
+			return nil
+		}
+	case time.Time:
+		switch d := dest.(type) {
+		case *string:
+			*d = s.Format(time.RFC3339Nano)
+			return nil
+		case *[]byte:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = []byte(s.Format(time.RFC3339Nano))
+			return nil
+		}
+	case nil:
+		switch d := dest.(type) {
+		case *interface{}:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = nil
+			return nil
+		case *[]byte:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = nil
+			return nil
+		case *sql.RawBytes:
+			if d == nil {
+				return errNilPtr
+			}
+			*d = nil
+			return nil
+		}
+	}
+
+	var sv reflect.Value
+
+	switch d := dest.(type) {
+	case *string:
+		sv = reflect.ValueOf(src)
+		switch sv.Kind() {
+		case reflect.Bool,
+			reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64,
+			reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64,
+			reflect.Float32, reflect.Float64:
+			*d = asString(src)
+			return nil
+		}
+	case *[]byte:
+		sv = reflect.ValueOf(src)
+		if b, ok := asBytes(nil, sv); ok {
+			*d = b
+			return nil
+		}
+	case *sql.RawBytes:
+		sv = reflect.ValueOf(src)
+		if b, ok := asBytes([]byte(*d)[:0], sv); ok {
+			*d = sql.RawBytes(b)
+			return nil
+		}
+	case *bool:
+		bv, err := driver.Bool.ConvertValue(src)
+		if err == nil {
+			*d = bv.(bool)
+		}
+		return err
+	case *interface{}:
+		*d = src
+		return nil
+	}
+
+	if scanner, ok := dest.(sql.Scanner); ok {
+		return scanner.Scan(src)
+	}
+
+	dpv := reflect.ValueOf(dest)
+	if dpv.Kind() != reflect.Ptr {
+		return errors.New("destination not a pointer")
+	}
+	if dpv.IsNil() {
+		return errNilPtr
+	}
+
+	if !sv.IsValid() {
+		sv = reflect.ValueOf(src)
+	}
+
+	dv := reflect.Indirect(dpv)
+	if sv.IsValid() && sv.Type().AssignableTo(dv.Type()) {
+		dv.Set(sv)
+		return nil
+	}
+
+	if dv.Kind() == sv.Kind() && sv.Type().ConvertibleTo(dv.Type()) {
+		dv.Set(sv.Convert(dv.Type()))
+		return nil
+	}
+
+	switch dv.Kind() {
+	case reflect.Ptr:
+		if src == nil {
+			dv.Set(reflect.Zero(dv.Type()))
+			return nil
+		} else {
+			dv.Set(reflect.New(dv.Type().Elem()))
+			return ConvertAssign(dv.Interface(), src)
+		}
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		s := asString(src)
+		i64, err := strconv.ParseInt(s, 10, dv.Type().Bits())
+		if err != nil {
+			err = strconvErr(err)
+			return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err)
+		}
+		dv.SetInt(i64)
+		return nil
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		s := asString(src)
+		u64, err := strconv.ParseUint(s, 10, dv.Type().Bits())
+		if err != nil {
+			err = strconvErr(err)
+			return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err)
+		}
+		dv.SetUint(u64)
+		return nil
+	case reflect.Float32, reflect.Float64:
+		s := asString(src)
+		f64, err := strconv.ParseFloat(s, dv.Type().Bits())
+		if err != nil {
+			err = strconvErr(err)
+			return fmt.Errorf("converting driver.Value type %T (%q) to a %s: %v", src, s, dv.Kind(), err)
+		}
+		dv.SetFloat(f64)
+		return nil
+	}
+
+	return fmt.Errorf("unsupported Scan, storing driver.Value type %T into type %T", src, dest)
+}
+
+func strconvErr(err error) error {
+	if ne, ok := err.(*strconv.NumError); ok {
+		return ne.Err
+	}
+	return err
+}
+
+func cloneBytes(b []byte) []byte {
+	if b == nil {
+		return nil
+	} else {
+		c := make([]byte, len(b))
+		copy(c, b)
+		return c
+	}
+}
+
+func asString(src interface{}) string {
+	switch v := src.(type) {
+	case string:
+		return v
+	case []byte:
+		return string(v)
+	}
+	rv := reflect.ValueOf(src)
+	switch rv.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return strconv.FormatInt(rv.Int(), 10)
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return strconv.FormatUint(rv.Uint(), 10)
+	case reflect.Float64:
+		return strconv.FormatFloat(rv.Float(), 'g', -1, 64)
+	case reflect.Float32:
+		return strconv.FormatFloat(rv.Float(), 'g', -1, 32)
+	case reflect.Bool:
+		return strconv.FormatBool(rv.Bool())
+	}
+	return fmt.Sprintf("%v", src)
+}
+
+func asBytes(buf []byte, rv reflect.Value) (b []byte, ok bool) {
+	switch rv.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return strconv.AppendInt(buf, rv.Int(), 10), true
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		return strconv.AppendUint(buf, rv.Uint(), 10), true
+	case reflect.Float32:
+		return strconv.AppendFloat(buf, rv.Float(), 'g', -1, 32), true
+	case reflect.Float64:
+		return strconv.AppendFloat(buf, rv.Float(), 'g', -1, 64), true
+	case reflect.Bool:
+		return strconv.AppendBool(buf, rv.Bool()), true
+	case reflect.String:
+		s := rv.String()
+		return append(buf, s...), true
+	}
+	return
+}

extras/null/convert/convert_test.go

@@ -0,0 +1,382 @@
+// Copyright 2011 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// These functions are copied from database/sql/convert_test.go build 1.6.2
+
+package convert
+
+import (
+	"database/sql"
+	"database/sql/driver"
+	"fmt"
+	"reflect"
+	"runtime"
+	"testing"
+	"time"
+)
+
+var someTime = time.Unix(123, 0)
+var answer int64 = 42
+
+type userDefined float64
+
+type userDefinedSlice []int
+
+type conversionTest struct {
+	s, d interface{} // source and destination
+
+	// following are used if they're non-zero
+	wantint    int64
+	wantuint   uint64
+	wantstr    string
+	wantbytes  []byte
+	wantraw    sql.RawBytes
+	wantf32    float32
+	wantf64    float64
+	wanttime   time.Time
+	wantbool   bool // used if d is of type *bool
+	wanterr    string
+	wantiface  interface{}
+	wantptr    *int64 // if non-nil, *d's pointed value must be equal to *wantptr
+	wantnil    bool   // if true, *d must be *int64(nil)
+	wantusrdef userDefined
+}
+
+// Target variables for scanning into.
+var (
+	scanstr    string
+	scanbytes  []byte
+	scanraw    sql.RawBytes
+	scanint    int
+	scanint8   int8
+	scanint16  int16
+	scanint32  int32
+	scanuint8  uint8
+	scanuint16 uint16
+	scanbool   bool
+	scanf32    float32
+	scanf64    float64
+	scantime   time.Time
+	scanptr    *int64
+	scaniface  interface{}
+)
+
+var conversionTests = []conversionTest{
+	// Exact conversions (destination pointer type matches source type)
+	{s: "foo", d: &scanstr, wantstr: "foo"},
+	{s: 123, d: &scanint, wantint: 123},
+	{s: someTime, d: &scantime, wanttime: someTime},
+
+	// To strings
+	{s: "string", d: &scanstr, wantstr: "string"},
+	{s: []byte("byteslice"), d: &scanstr, wantstr: "byteslice"},
+	{s: 123, d: &scanstr, wantstr: "123"},
+	{s: int8(123), d: &scanstr, wantstr: "123"},
+	{s: int64(123), d: &scanstr, wantstr: "123"},
+	{s: uint8(123), d: &scanstr, wantstr: "123"},
+	{s: uint16(123), d: &scanstr, wantstr: "123"},
+	{s: uint32(123), d: &scanstr, wantstr: "123"},
+	{s: uint64(123), d: &scanstr, wantstr: "123"},
+	{s: 1.5, d: &scanstr, wantstr: "1.5"},
+
+	// From time.Time:
+	{s: time.Unix(1, 0).UTC(), d: &scanstr, wantstr: "1970-01-01T00:00:01Z"},
+	{s: time.Unix(1453874597, 0).In(time.FixedZone("here", -3600*8)), d: &scanstr, wantstr: "2016-01-26T22:03:17-08:00"},
+	{s: time.Unix(1, 2).UTC(), d: &scanstr, wantstr: "1970-01-01T00:00:01.000000002Z"},
+	{s: time.Time{}, d: &scanstr, wantstr: "0001-01-01T00:00:00Z"},
+	{s: time.Unix(1, 2).UTC(), d: &scanbytes, wantbytes: []byte("1970-01-01T00:00:01.000000002Z")},
+	{s: time.Unix(1, 2).UTC(), d: &scaniface, wantiface: time.Unix(1, 2).UTC()},
+
+	// To []byte
+	{s: nil, d: &scanbytes, wantbytes: nil},
+	{s: "string", d: &scanbytes, wantbytes: []byte("string")},
+	{s: []byte("byteslice"), d: &scanbytes, wantbytes: []byte("byteslice")},
+	{s: 123, d: &scanbytes, wantbytes: []byte("123")},
+	{s: int8(123), d: &scanbytes, wantbytes: []byte("123")},
+	{s: int64(123), d: &scanbytes, wantbytes: []byte("123")},
+	{s: uint8(123), d: &scanbytes, wantbytes: []byte("123")},
+	{s: uint16(123), d: &scanbytes, wantbytes: []byte("123")},
+	{s: uint32(123), d: &scanbytes, wantbytes: []byte("123")},
+	{s: uint64(123), d: &scanbytes, wantbytes: []byte("123")},
+	{s: 1.5, d: &scanbytes, wantbytes: []byte("1.5")},
+
+	// To sql.RawBytes
+	{s: nil, d: &scanraw, wantraw: nil},
+	{s: []byte("byteslice"), d: &scanraw, wantraw: sql.RawBytes("byteslice")},
+	{s: 123, d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: int8(123), d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: int64(123), d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: uint8(123), d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: uint16(123), d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: uint32(123), d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: uint64(123), d: &scanraw, wantraw: sql.RawBytes("123")},
+	{s: 1.5, d: &scanraw, wantraw: sql.RawBytes("1.5")},
+
+	// Strings to integers
+	{s: "255", d: &scanuint8, wantuint: 255},
+	{s: "256", d: &scanuint8, wanterr: "converting driver.Value type string (\"256\") to a uint8: value out of range"},
+	{s: "256", d: &scanuint16, wantuint: 256},
+	{s: "-1", d: &scanint, wantint: -1},
+	{s: "foo", d: &scanint, wanterr: "converting driver.Value type string (\"foo\") to a int: invalid syntax"},
+
+	// int64 to smaller integers
+	{s: int64(5), d: &scanuint8, wantuint: 5},
+	{s: int64(256), d: &scanuint8, wanterr: "converting driver.Value type int64 (\"256\") to a uint8: value out of range"},
+	{s: int64(256), d: &scanuint16, wantuint: 256},
+	{s: int64(65536), d: &scanuint16, wanterr: "converting driver.Value type int64 (\"65536\") to a uint16: value out of range"},
+
+	// True bools
+	{s: true, d: &scanbool, wantbool: true},
+	{s: "True", d: &scanbool, wantbool: true},
+	{s: "TRUE", d: &scanbool, wantbool: true},
+	{s: "1", d: &scanbool, wantbool: true},
+	{s: 1, d: &scanbool, wantbool: true},
+	{s: int64(1), d: &scanbool, wantbool: true},
+	{s: uint16(1), d: &scanbool, wantbool: true},
+
+	// False bools
+	{s: false, d: &scanbool, wantbool: false},
+	{s: "false", d: &scanbool, wantbool: false},
+	{s: "FALSE", d: &scanbool, wantbool: false},
+	{s: "0", d: &scanbool, wantbool: false},
+	{s: 0, d: &scanbool, wantbool: false},
+	{s: int64(0), d: &scanbool, wantbool: false},
+	{s: uint16(0), d: &scanbool, wantbool: false},
+
+	// Not bools
+	{s: "yup", d: &scanbool, wanterr: `sql/driver: couldn't convert "yup" into type bool`},
+	{s: 2, d: &scanbool, wanterr: `sql/driver: couldn't convert 2 into type bool`},
+
+	// Floats
+	{s: float64(1.5), d: &scanf64, wantf64: float64(1.5)},
+	{s: int64(1), d: &scanf64, wantf64: float64(1)},
+	{s: float64(1.5), d: &scanf32, wantf32: float32(1.5)},
+	{s: "1.5", d: &scanf32, wantf32: float32(1.5)},
+	{s: "1.5", d: &scanf64, wantf64: float64(1.5)},
+
+	// Pointers
+	{s: interface{}(nil), d: &scanptr, wantnil: true},
+	{s: int64(42), d: &scanptr, wantptr: &answer},
+
+	// To interface{}
+	{s: float64(1.5), d: &scaniface, wantiface: float64(1.5)},
+	{s: int64(1), d: &scaniface, wantiface: int64(1)},
+	{s: "str", d: &scaniface, wantiface: "str"},
+	{s: []byte("byteslice"), d: &scaniface, wantiface: []byte("byteslice")},
+	{s: true, d: &scaniface, wantiface: true},
+	{s: nil, d: &scaniface},
+	{s: []byte(nil), d: &scaniface, wantiface: []byte(nil)},
+
+	// To a user-defined type
+	{s: 1.5, d: new(userDefined), wantusrdef: 1.5},
+	{s: int64(123), d: new(userDefined), wantusrdef: 123},
+	{s: "1.5", d: new(userDefined), wantusrdef: 1.5},
+	{s: []byte{1, 2, 3}, d: new(userDefinedSlice), wanterr: `unsupported Scan, storing driver.Value type []uint8 into type *convert.userDefinedSlice`},
+
+	// Other errors
+	{s: complex(1, 2), d: &scanstr, wanterr: `unsupported Scan, storing driver.Value type complex128 into type *string`},
+}
+
+func intPtrValue(intptr interface{}) interface{} {
+	return reflect.Indirect(reflect.Indirect(reflect.ValueOf(intptr))).Int()
+}
+
+func intValue(intptr interface{}) int64 {
+	return reflect.Indirect(reflect.ValueOf(intptr)).Int()
+}
+
+func uintValue(intptr interface{}) uint64 {
+	return reflect.Indirect(reflect.ValueOf(intptr)).Uint()
+}
+
+func float64Value(ptr interface{}) float64 {
+	return *(ptr.(*float64))
+}
+
+func float32Value(ptr interface{}) float32 {
+	return *(ptr.(*float32))
+}
+
+func getTimeValue(ptr interface{}) time.Time {
+	return *(ptr.(*time.Time))
+}
+
+func TestConversions(t *testing.T) {
+	for n, ct := range conversionTests {
+		err := ConvertAssign(ct.d, ct.s)
+		errstr := ""
+		if err != nil {
+			errstr = err.Error()
+		}
+		errf := func(format string, args ...interface{}) {
+			base := fmt.Sprintf("ConvertAssign #%d: for %v (%T) -> %T, ", n, ct.s, ct.s, ct.d)
+			t.Errorf(base+format, args...)
+		}
+		if errstr != ct.wanterr {
+			errf("got error %q, want error %q", errstr, ct.wanterr)
+		}
+		if ct.wantstr != "" && ct.wantstr != scanstr {
+			errf("want string %q, got %q", ct.wantstr, scanstr)
+		}
+		if ct.wantint != 0 && ct.wantint != intValue(ct.d) {
+			errf("want int %d, got %d", ct.wantint, intValue(ct.d))
+		}
+		if ct.wantuint != 0 && ct.wantuint != uintValue(ct.d) {
+			errf("want uint %d, got %d", ct.wantuint, uintValue(ct.d))
+		}
+		if ct.wantf32 != 0 && ct.wantf32 != float32Value(ct.d) {
+			errf("want float32 %v, got %v", ct.wantf32, float32Value(ct.d))
+		}
+		if ct.wantf64 != 0 && ct.wantf64 != float64Value(ct.d) {
+			errf("want float32 %v, got %v", ct.wantf64, float64Value(ct.d))
+		}
+		if bp, boolTest := ct.d.(*bool); boolTest && *bp != ct.wantbool && ct.wanterr == "" {
+			errf("want bool %v, got %v", ct.wantbool, *bp)
+		}
+		if !ct.wanttime.IsZero() && !ct.wanttime.Equal(getTimeValue(ct.d)) {
+			errf("want time %v, got %v", ct.wanttime, getTimeValue(ct.d))
+		}
+		if ct.wantnil && *ct.d.(**int64) != nil {
+			errf("want nil, got %v", intPtrValue(ct.d))
+		}
+		if ct.wantptr != nil {
+			if *ct.d.(**int64) == nil {
+				errf("want pointer to %v, got nil", *ct.wantptr)
+			} else if *ct.wantptr != intPtrValue(ct.d) {
+				errf("want pointer to %v, got %v", *ct.wantptr, intPtrValue(ct.d))
+			}
+		}
+		if ifptr, ok := ct.d.(*interface{}); ok {
+			if !reflect.DeepEqual(ct.wantiface, scaniface) {
+				errf("want interface %#v, got %#v", ct.wantiface, scaniface)
+				continue
+			}
+			if srcBytes, ok := ct.s.([]byte); ok {
+				dstBytes := (*ifptr).([]byte)
+				if len(srcBytes) > 0 && &dstBytes[0] == &srcBytes[0] {
+					errf("copy into interface{} didn't copy []byte data")
+				}
+			}
+		}
+		if ct.wantusrdef != 0 && ct.wantusrdef != *ct.d.(*userDefined) {
+			errf("want userDefined %f, got %f", ct.wantusrdef, *ct.d.(*userDefined))
+		}
+	}
+}
+
+func TestNullString(t *testing.T) {
+	var ns sql.NullString
+	ConvertAssign(&ns, []byte("foo"))
+	if !ns.Valid {
+		t.Errorf("expecting not null")
+	}
+	if ns.String != "foo" {
+		t.Errorf("expecting foo; got %q", ns.String)
+	}
+	ConvertAssign(&ns, nil)
+	if ns.Valid {
+		t.Errorf("expecting null on nil")
+	}
+	if ns.String != "" {
+		t.Errorf("expecting blank on nil; got %q", ns.String)
+	}
+}
+
+type valueConverterTest struct {
+	c       driver.ValueConverter
+	in, out interface{}
+	err     string
+}
+
+var valueConverterTests = []valueConverterTest{
+	{driver.DefaultParameterConverter, sql.NullString{"hi", true}, "hi", ""},
+	{driver.DefaultParameterConverter, sql.NullString{"", false}, nil, ""},
+}
+
+func TestValueConverters(t *testing.T) {
+	for i, tt := range valueConverterTests {
+		out, err := tt.c.ConvertValue(tt.in)
+		goterr := ""
+		if err != nil {
+			goterr = err.Error()
+		}
+		if goterr != tt.err {
+			t.Errorf("test %d: %T(%T(%v)) error = %q; want error = %q",
+				i, tt.c, tt.in, tt.in, goterr, tt.err)
+		}
+		if tt.err != "" {
+			continue
+		}
+		if !reflect.DeepEqual(out, tt.out) {
+			t.Errorf("test %d: %T(%T(%v)) = %v (%T); want %v (%T)",
+				i, tt.c, tt.in, tt.in, out, out, tt.out, tt.out)
+		}
+	}
+}
+
+// Tests that assigning to sql.RawBytes doesn't allocate (and also works).
+func TestRawBytesAllocs(t *testing.T) {
+	var tests = []struct {
+		name string
+		in   interface{}
+		want string
+	}{
+		{"uint64", uint64(12345678), "12345678"},
+		{"uint32", uint32(1234), "1234"},
+		{"uint16", uint16(12), "12"},
+		{"uint8", uint8(1), "1"},
+		{"uint", uint(123), "123"},
+		{"int", int(123), "123"},
+		{"int8", int8(1), "1"},
+		{"int16", int16(12), "12"},
+		{"int32", int32(1234), "1234"},
+		{"int64", int64(12345678), "12345678"},
+		{"float32", float32(1.5), "1.5"},
+		{"float64", float64(64), "64"},
+		{"bool", false, "false"},
+	}
+
+	buf := make(sql.RawBytes, 10)
+	test := func(name string, in interface{}, want string) {
+		if err := ConvertAssign(&buf, in); err != nil {
+			t.Fatalf("%s: ConvertAssign = %v", name, err)
+		}
+		match := len(buf) == len(want)
+		if match {
+			for i, b := range buf {
+				if want[i] != b {
+					match = false
+					break
+				}
+			}
+		}
+		if !match {
+			t.Fatalf("%s: got %q (len %d); want %q (len %d)", name, buf, len(buf), want, len(want))
+		}
+	}
+
+	n := testing.AllocsPerRun(100, func() {
+		for _, tt := range tests {
+			test(tt.name, tt.in, tt.want)
+		}
+	})
+
+	// The numbers below are only valid for 64-bit interface word sizes,
+	// and gc. With 32-bit words there are more convT2E allocs, and
+	// with gccgo, only pointers currently go in interface data.
+	// So only care on amd64 gc for now.
+	measureAllocs := runtime.GOARCH == "amd64" && runtime.Compiler == "gc"
+
+	if n > 0.5 && measureAllocs {
+		t.Fatalf("allocs = %v; want 0", n)
+	}
+
+	// This one involves a convT2E allocation, string -> interface{}
+	n = testing.AllocsPerRun(100, func() {
+		test("string", "foo", "foo")
+	})
+	if n > 1.5 && measureAllocs {
+		t.Fatalf("allocs = %v; want max 1", n)
+	}
+}

extras/null/float32.go

@@ -0,0 +1,123 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Float32 is a nullable float32.
+type Float32 struct {
+	Float32 float32
+	Valid   bool
+}
+
+// NewFloat32 creates a new Float32
+func NewFloat32(f float32, valid bool) Float32 {
+	return Float32{
+		Float32: f,
+		Valid:   valid,
+	}
+}
+
+// Float32From creates a new Float32 that will always be valid.
+func Float32From(f float32) Float32 {
+	return NewFloat32(f, true)
+}
+
+// Float32FromPtr creates a new Float32 that be null if f is nil.
+func Float32FromPtr(f *float32) Float32 {
+	if f == nil {
+		return NewFloat32(0, false)
+	}
+	return NewFloat32(*f, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (f *Float32) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		f.Valid = false
+		f.Float32 = 0
+		return nil
+	}
+
+	var x float64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	f.Float32 = float32(x)
+	f.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (f *Float32) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		f.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseFloat(string(text), 32)
+	f.Valid = err == nil
+	if f.Valid {
+		f.Float32 = float32(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (f Float32) MarshalJSON() ([]byte, error) {
+	if !f.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatFloat(float64(f.Float32), 'f', -1, 32)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (f Float32) MarshalText() ([]byte, error) {
+	if !f.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatFloat(float64(f.Float32), 'f', -1, 32)), nil
+}
+
+// SetValid changes this Float32's value and also sets it to be non-null.
+func (f *Float32) SetValid(n float32) {
+	f.Float32 = n
+	f.Valid = true
+}
+
+// Ptr returns a pointer to this Float32's value, or a nil pointer if this Float32 is null.
+func (f Float32) Ptr() *float32 {
+	if !f.Valid {
+		return nil
+	}
+	return &f.Float32
+}
+
+// IsNull returns true for invalid Float32s, for future omitempty support (Go 1.4?)
+func (f Float32) IsNull() bool {
+	return !f.Valid
+}
+
+// Scan implements the Scanner interface.
+func (f *Float32) Scan(value interface{}) error {
+	if value == nil {
+		f.Float32, f.Valid = 0, false
+		return nil
+	}
+	f.Valid = true
+	return convert.ConvertAssign(&f.Float32, value)
+}
+
+// Value implements the driver Valuer interface.
+func (f Float32) Value() (driver.Value, error) {
+	if !f.Valid {
+		return nil, nil
+	}
+	return float64(f.Float32), nil
+}

extras/null/float32_test.go

@@ -0,0 +1,164 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	float32JSON = []byte(`1.2345`)
+)
+
+func TestFloat32From(t *testing.T) {
+	f := Float32From(1.2345)
+	assertFloat32(t, f, "Float32From()")
+
+	zero := Float32From(0)
+	if !zero.Valid {
+		t.Error("Float32From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestFloat32FromPtr(t *testing.T) {
+	n := float32(1.2345)
+	iptr := &n
+	f := Float32FromPtr(iptr)
+	assertFloat32(t, f, "Float32FromPtr()")
+
+	null := Float32FromPtr(nil)
+	assertNullFloat32(t, null, "Float32FromPtr(nil)")
+}
+
+func TestUnmarshalFloat32(t *testing.T) {
+	var f Float32
+	err := json.Unmarshal(float32JSON, &f)
+	maybePanic(err)
+	assertFloat32(t, f, "float32 json")
+
+	var null Float32
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullFloat32(t, null, "null json")
+
+	var badType Float32
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullFloat32(t, badType, "wrong type json")
+
+	var invalid Float32
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+}
+
+func TestTextUnmarshalFloat32(t *testing.T) {
+	var f Float32
+	err := f.UnmarshalText([]byte("1.2345"))
+	maybePanic(err)
+	assertFloat32(t, f, "UnmarshalText() float32")
+
+	var blank Float32
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullFloat32(t, blank, "UnmarshalText() empty float32")
+}
+
+func TestMarshalFloat32(t *testing.T) {
+	f := Float32From(1.2345)
+	data, err := json.Marshal(f)
+	maybePanic(err)
+	assertJSONEquals(t, data, "1.2345", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewFloat32(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalFloat32Text(t *testing.T) {
+	f := Float32From(1.2345)
+	data, err := f.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "1.2345", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewFloat32(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestFloat32Pointer(t *testing.T) {
+	f := Float32From(1.2345)
+	ptr := f.Ptr()
+	if *ptr != 1.2345 {
+		t.Errorf("bad %s float32: %#v ≠ %v\n", "pointer", ptr, 1.2345)
+	}
+
+	null := NewFloat32(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s float32: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestFloat32IsNull(t *testing.T) {
+	f := Float32From(1.2345)
+	if f.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewFloat32(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewFloat32(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestFloat32SetValid(t *testing.T) {
+	change := NewFloat32(0, false)
+	assertNullFloat32(t, change, "SetValid()")
+	change.SetValid(1.2345)
+	assertFloat32(t, change, "SetValid()")
+}
+
+func TestFloat32Scan(t *testing.T) {
+	var f Float32
+	err := f.Scan(1.2345)
+	maybePanic(err)
+	assertFloat32(t, f, "scanned float32")
+
+	var null Float32
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullFloat32(t, null, "scanned null")
+}
+
+func assertFloat32(t *testing.T, f Float32, from string) {
+	if f.Float32 != 1.2345 {
+		t.Errorf("bad %s float32: %f ≠ %f\n", from, f.Float32, 1.2345)
+	}
+	if !f.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullFloat32(t *testing.T, f Float32, from string) {
+	if f.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/float64.go

@@ -0,0 +1,118 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Float64 is a nullable float64.
+type Float64 struct {
+	Float64 float64
+	Valid   bool
+}
+
+// NewFloat64 creates a new Float64
+func NewFloat64(f float64, valid bool) Float64 {
+	return Float64{
+		Float64: f,
+		Valid:   valid,
+	}
+}
+
+// Float64From creates a new Float64 that will always be valid.
+func Float64From(f float64) Float64 {
+	return NewFloat64(f, true)
+}
+
+// Float64FromPtr creates a new Float64 that be null if f is nil.
+func Float64FromPtr(f *float64) Float64 {
+	if f == nil {
+		return NewFloat64(0, false)
+	}
+	return NewFloat64(*f, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (f *Float64) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		f.Float64 = 0
+		f.Valid = false
+		return nil
+	}
+
+	if err := json.Unmarshal(data, &f.Float64); err != nil {
+		return err
+	}
+
+	f.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (f *Float64) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		f.Valid = false
+		return nil
+	}
+	var err error
+	f.Float64, err = strconv.ParseFloat(string(text), 64)
+	f.Valid = err == nil
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (f Float64) MarshalJSON() ([]byte, error) {
+	if !f.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatFloat(f.Float64, 'f', -1, 64)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (f Float64) MarshalText() ([]byte, error) {
+	if !f.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatFloat(f.Float64, 'f', -1, 64)), nil
+}
+
+// SetValid changes this Float64's value and also sets it to be non-null.
+func (f *Float64) SetValid(n float64) {
+	f.Float64 = n
+	f.Valid = true
+}
+
+// Ptr returns a pointer to this Float64's value, or a nil pointer if this Float64 is null.
+func (f Float64) Ptr() *float64 {
+	if !f.Valid {
+		return nil
+	}
+	return &f.Float64
+}
+
+// IsNull returns true for invalid Float64s, for future omitempty support (Go 1.4?)
+func (f Float64) IsNull() bool {
+	return !f.Valid
+}
+
+// Scan implements the Scanner interface.
+func (f *Float64) Scan(value interface{}) error {
+	if value == nil {
+		f.Float64, f.Valid = 0, false
+		return nil
+	}
+	f.Valid = true
+	return convert.ConvertAssign(&f.Float64, value)
+}
+
+// Value implements the driver Valuer interface.
+func (f Float64) Value() (driver.Value, error) {
+	if !f.Valid {
+		return nil, nil
+	}
+	return f.Float64, nil
+}

extras/null/float64_test.go

@@ -0,0 +1,164 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	float64JSON = []byte(`1.2345`)
+)
+
+func TestFloat64From(t *testing.T) {
+	f := Float64From(1.2345)
+	assertFloat64(t, f, "Float64From()")
+
+	zero := Float64From(0)
+	if !zero.Valid {
+		t.Error("Float64From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestFloat64FromPtr(t *testing.T) {
+	n := float64(1.2345)
+	iptr := &n
+	f := Float64FromPtr(iptr)
+	assertFloat64(t, f, "Float64FromPtr()")
+
+	null := Float64FromPtr(nil)
+	assertNullFloat64(t, null, "Float64FromPtr(nil)")
+}
+
+func TestUnmarshalFloat64(t *testing.T) {
+	var f Float64
+	err := json.Unmarshal(float64JSON, &f)
+	maybePanic(err)
+	assertFloat64(t, f, "float64 json")
+
+	var null Float64
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullFloat64(t, null, "null json")
+
+	var badType Float64
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullFloat64(t, badType, "wrong type json")
+
+	var invalid Float64
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+}
+
+func TestTextUnmarshalFloat64(t *testing.T) {
+	var f Float64
+	err := f.UnmarshalText([]byte("1.2345"))
+	maybePanic(err)
+	assertFloat64(t, f, "UnmarshalText() float64")
+
+	var blank Float64
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullFloat64(t, blank, "UnmarshalText() empty float64")
+}
+
+func TestMarshalFloat64(t *testing.T) {
+	f := Float64From(1.2345)
+	data, err := json.Marshal(f)
+	maybePanic(err)
+	assertJSONEquals(t, data, "1.2345", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewFloat64(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalFloat64Text(t *testing.T) {
+	f := Float64From(1.2345)
+	data, err := f.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "1.2345", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewFloat64(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestFloat64Pointer(t *testing.T) {
+	f := Float64From(1.2345)
+	ptr := f.Ptr()
+	if *ptr != 1.2345 {
+		t.Errorf("bad %s float64: %#v ≠ %v\n", "pointer", ptr, 1.2345)
+	}
+
+	null := NewFloat64(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s float64: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestFloat64IsNull(t *testing.T) {
+	f := Float64From(1.2345)
+	if f.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewFloat64(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewFloat64(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestFloat64SetValid(t *testing.T) {
+	change := NewFloat64(0, false)
+	assertNullFloat64(t, change, "SetValid()")
+	change.SetValid(1.2345)
+	assertFloat64(t, change, "SetValid()")
+}
+
+func TestFloat64Scan(t *testing.T) {
+	var f Float64
+	err := f.Scan(1.2345)
+	maybePanic(err)
+	assertFloat64(t, f, "scanned float64")
+
+	var null Float64
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullFloat64(t, null, "scanned null")
+}
+
+func assertFloat64(t *testing.T, f Float64, from string) {
+	if f.Float64 != 1.2345 {
+		t.Errorf("bad %s float64: %f ≠ %f\n", from, f.Float64, 1.2345)
+	}
+	if !f.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullFloat64(t *testing.T, f Float64, from string) {
+	if f.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/int.go

@@ -0,0 +1,123 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Int is an nullable int.
+type Int struct {
+	Int   int
+	Valid bool
+}
+
+// NewInt creates a new Int
+func NewInt(i int, valid bool) Int {
+	return Int{
+		Int:   i,
+		Valid: valid,
+	}
+}
+
+// IntFrom creates a new Int that will always be valid.
+func IntFrom(i int) Int {
+	return NewInt(i, true)
+}
+
+// IntFromPtr creates a new Int that be null if i is nil.
+func IntFromPtr(i *int) Int {
+	if i == nil {
+		return NewInt(0, false)
+	}
+	return NewInt(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (i *Int) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		i.Valid = false
+		i.Int = 0
+		return nil
+	}
+
+	var x int64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	i.Int = int(x)
+	i.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (i *Int) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		i.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseInt(string(text), 10, 0)
+	i.Valid = err == nil
+	if i.Valid {
+		i.Int = int(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (i Int) MarshalJSON() ([]byte, error) {
+	if !i.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (i Int) MarshalText() ([]byte, error) {
+	if !i.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int), 10)), nil
+}
+
+// SetValid changes this Int's value and also sets it to be non-null.
+func (i *Int) SetValid(n int) {
+	i.Int = n
+	i.Valid = true
+}
+
+// Ptr returns a pointer to this Int's value, or a nil pointer if this Int is null.
+func (i Int) Ptr() *int {
+	if !i.Valid {
+		return nil
+	}
+	return &i.Int
+}
+
+// IsNull returns true for invalid Ints, for future omitempty support (Go 1.4?)
+func (i Int) IsNull() bool {
+	return !i.Valid
+}
+
+// Scan implements the Scanner interface.
+func (i *Int) Scan(value interface{}) error {
+	if value == nil {
+		i.Int, i.Valid = 0, false
+		return nil
+	}
+	i.Valid = true
+	return convert.ConvertAssign(&i.Int, value)
+}
+
+// Value implements the driver Valuer interface.
+func (i Int) Value() (driver.Value, error) {
+	if !i.Valid {
+		return nil, nil
+	}
+	return int64(i.Int), nil
+}

extras/null/int16.go

@@ -0,0 +1,129 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Int16 is an nullable int16.
+type Int16 struct {
+	Int16 int16
+	Valid bool
+}
+
+// NewInt16 creates a new Int16
+func NewInt16(i int16, valid bool) Int16 {
+	return Int16{
+		Int16: i,
+		Valid: valid,
+	}
+}
+
+// Int16From creates a new Int16 that will always be valid.
+func Int16From(i int16) Int16 {
+	return NewInt16(i, true)
+}
+
+// Int16FromPtr creates a new Int16 that be null if i is nil.
+func Int16FromPtr(i *int16) Int16 {
+	if i == nil {
+		return NewInt16(0, false)
+	}
+	return NewInt16(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (i *Int16) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		i.Valid = false
+		i.Int16 = 0
+		return nil
+	}
+
+	var x int64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if x > math.MaxInt16 {
+		return fmt.Errorf("json: %d overflows max int16 value", x)
+	}
+
+	i.Int16 = int16(x)
+	i.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (i *Int16) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		i.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseInt(string(text), 10, 16)
+	i.Valid = err == nil
+	if i.Valid {
+		i.Int16 = int16(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (i Int16) MarshalJSON() ([]byte, error) {
+	if !i.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int16), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (i Int16) MarshalText() ([]byte, error) {
+	if !i.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int16), 10)), nil
+}
+
+// SetValid changes this Int16's value and also sets it to be non-null.
+func (i *Int16) SetValid(n int16) {
+	i.Int16 = n
+	i.Valid = true
+}
+
+// Ptr returns a pointer to this Int16's value, or a nil pointer if this Int16 is null.
+func (i Int16) Ptr() *int16 {
+	if !i.Valid {
+		return nil
+	}
+	return &i.Int16
+}
+
+// IsNull returns true for invalid Int16's, for future omitempty support (Go 1.4?)
+func (i Int16) IsNull() bool {
+	return !i.Valid
+}
+
+// Scan implements the Scanner interface.
+func (i *Int16) Scan(value interface{}) error {
+	if value == nil {
+		i.Int16, i.Valid = 0, false
+		return nil
+	}
+	i.Valid = true
+	return convert.ConvertAssign(&i.Int16, value)
+}
+
+// Value implements the driver Valuer interface.
+func (i Int16) Value() (driver.Value, error) {
+	if !i.Valid {
+		return nil, nil
+	}
+	return int64(i.Int16), nil
+}

extras/null/int16_test.go

@@ -0,0 +1,190 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	int16JSON = []byte(`32766`)
+)
+
+func TestInt16From(t *testing.T) {
+	i := Int16From(32766)
+	assertInt16(t, i, "Int16From()")
+
+	zero := Int16From(0)
+	if !zero.Valid {
+		t.Error("Int16From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestInt16FromPtr(t *testing.T) {
+	n := int16(32766)
+	iptr := &n
+	i := Int16FromPtr(iptr)
+	assertInt16(t, i, "Int16FromPtr()")
+
+	null := Int16FromPtr(nil)
+	assertNullInt16(t, null, "Int16FromPtr(nil)")
+}
+
+func TestUnmarshalInt16(t *testing.T) {
+	var i Int16
+	err := json.Unmarshal(int16JSON, &i)
+	maybePanic(err)
+	assertInt16(t, i, "int16 json")
+
+	var null Int16
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullInt16(t, null, "null json")
+
+	var badType Int16
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullInt16(t, badType, "wrong type json")
+
+	var invalid Int16
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullInt16(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonIntegerNumber16(t *testing.T) {
+	var i Int16
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to int16")
+	}
+}
+
+func TestUnmarshalInt16Overflow(t *testing.T) {
+	int16Overflow := uint16(math.MaxInt16)
+
+	// Max int16 should decode successfully
+	var i Int16
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(int16Overflow), 10)), &i)
+	maybePanic(err)
+	// Attempt to overflow
+	int16Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(int16Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows int16")
+	}
+}
+
+func TestTextUnmarshalInt16(t *testing.T) {
+	var i Int16
+	err := i.UnmarshalText([]byte("32766"))
+	maybePanic(err)
+	assertInt16(t, i, "UnmarshalText() int16")
+
+	var blank Int16
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullInt16(t, blank, "UnmarshalText() empty int16")
+}
+
+func TestMarshalInt16(t *testing.T) {
+	i := Int16From(32766)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "32766", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt16(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalInt16Text(t *testing.T) {
+	i := Int16From(32766)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "32766", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt16(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestInt16Pointer(t *testing.T) {
+	i := Int16From(32766)
+	ptr := i.Ptr()
+	if *ptr != 32766 {
+		t.Errorf("bad %s int16: %#v ≠ %d\n", "pointer", ptr, 32766)
+	}
+
+	null := NewInt16(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s int16: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestInt16IsNull(t *testing.T) {
+	i := Int16From(32766)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewInt16(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewInt16(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestInt16SetValid(t *testing.T) {
+	change := NewInt16(0, false)
+	assertNullInt16(t, change, "SetValid()")
+	change.SetValid(32766)
+	assertInt16(t, change, "SetValid()")
+}
+
+func TestInt16Scan(t *testing.T) {
+	var i Int16
+	err := i.Scan(32766)
+	maybePanic(err)
+	assertInt16(t, i, "scanned int16")
+
+	var null Int16
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullInt16(t, null, "scanned null")
+}
+
+func assertInt16(t *testing.T, i Int16, from string) {
+	if i.Int16 != 32766 {
+		t.Errorf("bad %s int16: %d ≠ %d\n", from, i.Int16, 32766)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullInt16(t *testing.T, i Int16, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/int32.go

@@ -0,0 +1,129 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Int32 is an nullable int32.
+type Int32 struct {
+	Int32 int32
+	Valid bool
+}
+
+// NewInt32 creates a new Int32
+func NewInt32(i int32, valid bool) Int32 {
+	return Int32{
+		Int32: i,
+		Valid: valid,
+	}
+}
+
+// Int32From creates a new Int32 that will always be valid.
+func Int32From(i int32) Int32 {
+	return NewInt32(i, true)
+}
+
+// Int32FromPtr creates a new Int32 that be null if i is nil.
+func Int32FromPtr(i *int32) Int32 {
+	if i == nil {
+		return NewInt32(0, false)
+	}
+	return NewInt32(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (i *Int32) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		i.Valid = false
+		i.Int32 = 0
+		return nil
+	}
+
+	var x int64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if x > math.MaxInt32 {
+		return fmt.Errorf("json: %d overflows max int32 value", x)
+	}
+
+	i.Int32 = int32(x)
+	i.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (i *Int32) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		i.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseInt(string(text), 10, 32)
+	i.Valid = err == nil
+	if i.Valid {
+		i.Int32 = int32(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (i Int32) MarshalJSON() ([]byte, error) {
+	if !i.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int32), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (i Int32) MarshalText() ([]byte, error) {
+	if !i.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int32), 10)), nil
+}
+
+// SetValid changes this Int32's value and also sets it to be non-null.
+func (i *Int32) SetValid(n int32) {
+	i.Int32 = n
+	i.Valid = true
+}
+
+// Ptr returns a pointer to this Int32's value, or a nil pointer if this Int32 is null.
+func (i Int32) Ptr() *int32 {
+	if !i.Valid {
+		return nil
+	}
+	return &i.Int32
+}
+
+// IsNull returns true for invalid Int32's, for future omitempty support (Go 1.4?)
+func (i Int32) IsNull() bool {
+	return !i.Valid
+}
+
+// Scan implements the Scanner interface.
+func (i *Int32) Scan(value interface{}) error {
+	if value == nil {
+		i.Int32, i.Valid = 0, false
+		return nil
+	}
+	i.Valid = true
+	return convert.ConvertAssign(&i.Int32, value)
+}
+
+// Value implements the driver Valuer interface.
+func (i Int32) Value() (driver.Value, error) {
+	if !i.Valid {
+		return nil, nil
+	}
+	return int64(i.Int32), nil
+}

extras/null/int32_test.go

@@ -0,0 +1,191 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	int32JSON = []byte(`2147483646`)
+)
+
+func TestInt32From(t *testing.T) {
+	i := Int32From(2147483646)
+	assertInt32(t, i, "Int32From()")
+
+	zero := Int32From(0)
+	if !zero.Valid {
+		t.Error("Int32From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestInt32FromPtr(t *testing.T) {
+	n := int32(2147483646)
+	iptr := &n
+	i := Int32FromPtr(iptr)
+	assertInt32(t, i, "Int32FromPtr()")
+
+	null := Int32FromPtr(nil)
+	assertNullInt32(t, null, "Int32FromPtr(nil)")
+}
+
+func TestUnmarshalInt32(t *testing.T) {
+	var i Int32
+	err := json.Unmarshal(int32JSON, &i)
+	maybePanic(err)
+	assertInt32(t, i, "int32 json")
+
+	var null Int32
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullInt32(t, null, "null json")
+
+	var badType Int32
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullInt32(t, badType, "wrong type json")
+
+	var invalid Int32
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullInt32(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonIntegerNumber32(t *testing.T) {
+	var i Int32
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to int32")
+	}
+}
+
+func TestUnmarshalInt32Overflow(t *testing.T) {
+	int32Overflow := uint32(math.MaxInt32)
+
+	// Max int32 should decode successfully
+	var i Int32
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(int32Overflow), 10)), &i)
+	maybePanic(err)
+
+	// Attempt to overflow
+	int32Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(int32Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows int32")
+	}
+}
+
+func TestTextUnmarshalInt32(t *testing.T) {
+	var i Int32
+	err := i.UnmarshalText([]byte("2147483646"))
+	maybePanic(err)
+	assertInt32(t, i, "UnmarshalText() int32")
+
+	var blank Int32
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullInt32(t, blank, "UnmarshalText() empty int32")
+}
+
+func TestMarshalInt32(t *testing.T) {
+	i := Int32From(2147483646)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "2147483646", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt32(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalInt32Text(t *testing.T) {
+	i := Int32From(2147483646)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "2147483646", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt32(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestInt32Pointer(t *testing.T) {
+	i := Int32From(2147483646)
+	ptr := i.Ptr()
+	if *ptr != 2147483646 {
+		t.Errorf("bad %s int32: %#v ≠ %d\n", "pointer", ptr, 2147483646)
+	}
+
+	null := NewInt32(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s int32: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestInt32IsNull(t *testing.T) {
+	i := Int32From(2147483646)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewInt32(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewInt32(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestInt32SetValid(t *testing.T) {
+	change := NewInt32(0, false)
+	assertNullInt32(t, change, "SetValid()")
+	change.SetValid(2147483646)
+	assertInt32(t, change, "SetValid()")
+}
+
+func TestInt32Scan(t *testing.T) {
+	var i Int32
+	err := i.Scan(2147483646)
+	maybePanic(err)
+	assertInt32(t, i, "scanned int32")
+
+	var null Int32
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullInt32(t, null, "scanned null")
+}
+
+func assertInt32(t *testing.T, i Int32, from string) {
+	if i.Int32 != 2147483646 {
+		t.Errorf("bad %s int32: %d ≠ %d\n", from, i.Int32, 2147483646)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullInt32(t *testing.T, i Int32, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/int64.go

@@ -0,0 +1,118 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Int64 is an nullable int64.
+type Int64 struct {
+	Int64 int64
+	Valid bool
+}
+
+// NewInt64 creates a new Int64
+func NewInt64(i int64, valid bool) Int64 {
+	return Int64{
+		Int64: i,
+		Valid: valid,
+	}
+}
+
+// Int64From creates a new Int64 that will always be valid.
+func Int64From(i int64) Int64 {
+	return NewInt64(i, true)
+}
+
+// Int64FromPtr creates a new Int64 that be null if i is nil.
+func Int64FromPtr(i *int64) Int64 {
+	if i == nil {
+		return NewInt64(0, false)
+	}
+	return NewInt64(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (i *Int64) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		i.Valid = false
+		i.Int64 = 0
+		return nil
+	}
+
+	if err := json.Unmarshal(data, &i.Int64); err != nil {
+		return err
+	}
+
+	i.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (i *Int64) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		i.Valid = false
+		return nil
+	}
+	var err error
+	i.Int64, err = strconv.ParseInt(string(text), 10, 64)
+	i.Valid = err == nil
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (i Int64) MarshalJSON() ([]byte, error) {
+	if !i.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatInt(i.Int64, 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (i Int64) MarshalText() ([]byte, error) {
+	if !i.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatInt(i.Int64, 10)), nil
+}
+
+// SetValid changes this Int64's value and also sets it to be non-null.
+func (i *Int64) SetValid(n int64) {
+	i.Int64 = n
+	i.Valid = true
+}
+
+// Ptr returns a pointer to this Int64's value, or a nil pointer if this Int64 is null.
+func (i Int64) Ptr() *int64 {
+	if !i.Valid {
+		return nil
+	}
+	return &i.Int64
+}
+
+// IsNull returns true for invalid Int64's, for future omitempty support (Go 1.4?)
+func (i Int64) IsNull() bool {
+	return !i.Valid
+}
+
+// Scan implements the Scanner interface.
+func (i *Int64) Scan(value interface{}) error {
+	if value == nil {
+		i.Int64, i.Valid = 0, false
+		return nil
+	}
+	i.Valid = true
+	return convert.ConvertAssign(&i.Int64, value)
+}
+
+// Value implements the driver Valuer interface.
+func (i Int64) Value() (driver.Value, error) {
+	if !i.Valid {
+		return nil, nil
+	}
+	return i.Int64, nil
+}

extras/null/int64_test.go

@@ -0,0 +1,191 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	int64JSON = []byte(`9223372036854775806`)
+)
+
+func TestInt64From(t *testing.T) {
+	i := Int64From(9223372036854775806)
+	assertInt64(t, i, "Int64From()")
+
+	zero := Int64From(0)
+	if !zero.Valid {
+		t.Error("Int64From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestInt64FromPtr(t *testing.T) {
+	n := int64(9223372036854775806)
+	iptr := &n
+	i := Int64FromPtr(iptr)
+	assertInt64(t, i, "Int64FromPtr()")
+
+	null := Int64FromPtr(nil)
+	assertNullInt64(t, null, "Int64FromPtr(nil)")
+}
+
+func TestUnmarshalInt64(t *testing.T) {
+	var i Int64
+	err := json.Unmarshal(int64JSON, &i)
+	maybePanic(err)
+	assertInt64(t, i, "int64 json")
+
+	var null Int64
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullInt64(t, null, "null json")
+
+	var badType Int64
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullInt64(t, badType, "wrong type json")
+
+	var invalid Int64
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullInt64(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonIntegerNumber64(t *testing.T) {
+	var i Int64
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to int64")
+	}
+}
+
+func TestUnmarshalInt64Overflow(t *testing.T) {
+	int64Overflow := uint64(math.MaxInt64)
+
+	// Max int64 should decode successfully
+	var i Int64
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(int64Overflow), 10)), &i)
+	maybePanic(err)
+
+	// Attempt to overflow
+	int64Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(int64Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows int64")
+	}
+}
+
+func TestTextUnmarshalInt64(t *testing.T) {
+	var i Int64
+	err := i.UnmarshalText([]byte("9223372036854775806"))
+	maybePanic(err)
+	assertInt64(t, i, "UnmarshalText() int64")
+
+	var blank Int64
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullInt64(t, blank, "UnmarshalText() empty int64")
+}
+
+func TestMarshalInt64(t *testing.T) {
+	i := Int64From(9223372036854775806)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "9223372036854775806", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt64(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalInt64Text(t *testing.T) {
+	i := Int64From(9223372036854775806)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "9223372036854775806", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt64(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestInt64Pointer(t *testing.T) {
+	i := Int64From(9223372036854775806)
+	ptr := i.Ptr()
+	if *ptr != 9223372036854775806 {
+		t.Errorf("bad %s int64: %#v ≠ %d\n", "pointer", ptr, 9223372036854775806)
+	}
+
+	null := NewInt64(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s int64: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestInt64IsNull(t *testing.T) {
+	i := Int64From(9223372036854775806)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewInt64(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewInt64(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestInt64SetValid(t *testing.T) {
+	change := NewInt64(0, false)
+	assertNullInt64(t, change, "SetValid()")
+	change.SetValid(9223372036854775806)
+	assertInt64(t, change, "SetValid()")
+}
+
+func TestInt64Scan(t *testing.T) {
+	var i Int64
+	err := i.Scan(9223372036854775806)
+	maybePanic(err)
+	assertInt64(t, i, "scanned int64")
+
+	var null Int64
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullInt64(t, null, "scanned null")
+}
+
+func assertInt64(t *testing.T, i Int64, from string) {
+	if i.Int64 != 9223372036854775806 {
+		t.Errorf("bad %s int64: %d ≠ %d\n", from, i.Int64, 9223372036854775806)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullInt64(t *testing.T, i Int64, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/int8.go

@@ -0,0 +1,129 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Int8 is an nullable int8.
+type Int8 struct {
+	Int8  int8
+	Valid bool
+}
+
+// NewInt8 creates a new Int8
+func NewInt8(i int8, valid bool) Int8 {
+	return Int8{
+		Int8:  i,
+		Valid: valid,
+	}
+}
+
+// Int8From creates a new Int8 that will always be valid.
+func Int8From(i int8) Int8 {
+	return NewInt8(i, true)
+}
+
+// Int8FromPtr creates a new Int8 that be null if i is nil.
+func Int8FromPtr(i *int8) Int8 {
+	if i == nil {
+		return NewInt8(0, false)
+	}
+	return NewInt8(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (i *Int8) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		i.Valid = false
+		i.Int8 = 0
+		return nil
+	}
+
+	var x int64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if x > math.MaxInt8 {
+		return fmt.Errorf("json: %d overflows max int8 value", x)
+	}
+
+	i.Int8 = int8(x)
+	i.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (i *Int8) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		i.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseInt(string(text), 10, 8)
+	i.Valid = err == nil
+	if i.Valid {
+		i.Int8 = int8(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (i Int8) MarshalJSON() ([]byte, error) {
+	if !i.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int8), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (i Int8) MarshalText() ([]byte, error) {
+	if !i.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatInt(int64(i.Int8), 10)), nil
+}
+
+// SetValid changes this Int8's value and also sets it to be non-null.
+func (i *Int8) SetValid(n int8) {
+	i.Int8 = n
+	i.Valid = true
+}
+
+// Ptr returns a pointer to this Int8's value, or a nil pointer if this Int8 is null.
+func (i Int8) Ptr() *int8 {
+	if !i.Valid {
+		return nil
+	}
+	return &i.Int8
+}
+
+// IsNull returns true for invalid Int8's, for future omitempty support (Go 1.4?)
+func (i Int8) IsNull() bool {
+	return !i.Valid
+}
+
+// Scan implements the Scanner interface.
+func (i *Int8) Scan(value interface{}) error {
+	if value == nil {
+		i.Int8, i.Valid = 0, false
+		return nil
+	}
+	i.Valid = true
+	return convert.ConvertAssign(&i.Int8, value)
+}
+
+// Value implements the driver Valuer interface.
+func (i Int8) Value() (driver.Value, error) {
+	if !i.Valid {
+		return nil, nil
+	}
+	return int64(i.Int8), nil
+}

extras/null/int8_test.go

@@ -0,0 +1,191 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	int8JSON = []byte(`126`)
+)
+
+func TestInt8From(t *testing.T) {
+	i := Int8From(126)
+	assertInt8(t, i, "Int8From()")
+
+	zero := Int8From(0)
+	if !zero.Valid {
+		t.Error("Int8From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestInt8FromPtr(t *testing.T) {
+	n := int8(126)
+	iptr := &n
+	i := Int8FromPtr(iptr)
+	assertInt8(t, i, "Int8FromPtr()")
+
+	null := Int8FromPtr(nil)
+	assertNullInt8(t, null, "Int8FromPtr(nil)")
+}
+
+func TestUnmarshalInt8(t *testing.T) {
+	var i Int8
+	err := json.Unmarshal(int8JSON, &i)
+	maybePanic(err)
+	assertInt8(t, i, "int8 json")
+
+	var null Int8
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullInt8(t, null, "null json")
+
+	var badType Int8
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullInt8(t, badType, "wrong type json")
+
+	var invalid Int8
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullInt8(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonIntegerNumber8(t *testing.T) {
+	var i Int8
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to int8")
+	}
+}
+
+func TestUnmarshalInt8Overflow(t *testing.T) {
+	int8Overflow := uint8(math.MaxInt8)
+
+	// Max int8 should decode successfully
+	var i Int8
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(int8Overflow), 10)), &i)
+	maybePanic(err)
+
+	// Attempt to overflow
+	int8Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(int8Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows int8")
+	}
+}
+
+func TestTextUnmarshalInt8(t *testing.T) {
+	var i Int8
+	err := i.UnmarshalText([]byte("126"))
+	maybePanic(err)
+	assertInt8(t, i, "UnmarshalText() int8")
+
+	var blank Int8
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullInt8(t, blank, "UnmarshalText() empty int8")
+}
+
+func TestMarshalInt8(t *testing.T) {
+	i := Int8From(126)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "126", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt8(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalInt8Text(t *testing.T) {
+	i := Int8From(126)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "126", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt8(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestInt8Pointer(t *testing.T) {
+	i := Int8From(126)
+	ptr := i.Ptr()
+	if *ptr != 126 {
+		t.Errorf("bad %s int8: %#v ≠ %d\n", "pointer", ptr, 126)
+	}
+
+	null := NewInt8(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s int8: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestInt8IsNull(t *testing.T) {
+	i := Int8From(126)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewInt8(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewInt8(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestInt8SetValid(t *testing.T) {
+	change := NewInt8(0, false)
+	assertNullInt8(t, change, "SetValid()")
+	change.SetValid(126)
+	assertInt8(t, change, "SetValid()")
+}
+
+func TestInt8Scan(t *testing.T) {
+	var i Int8
+	err := i.Scan(126)
+	maybePanic(err)
+	assertInt8(t, i, "scanned int8")
+
+	var null Int8
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullInt8(t, null, "scanned null")
+}
+
+func assertInt8(t *testing.T, i Int8, from string) {
+	if i.Int8 != 126 {
+		t.Errorf("bad %s int8: %d ≠ %d\n", from, i.Int8, 126)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullInt8(t *testing.T, i Int8, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/int_test.go

@@ -0,0 +1,173 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	intJSON = []byte(`12345`)
+)
+
+func TestIntFrom(t *testing.T) {
+	i := IntFrom(12345)
+	assertInt(t, i, "IntFrom()")
+
+	zero := IntFrom(0)
+	if !zero.Valid {
+		t.Error("IntFrom(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestIntFromPtr(t *testing.T) {
+	n := int(12345)
+	iptr := &n
+	i := IntFromPtr(iptr)
+	assertInt(t, i, "IntFromPtr()")
+
+	null := IntFromPtr(nil)
+	assertNullInt(t, null, "IntFromPtr(nil)")
+}
+
+func TestUnmarshalInt(t *testing.T) {
+	var i Int
+	err := json.Unmarshal(intJSON, &i)
+	maybePanic(err)
+	assertInt(t, i, "int json")
+
+	var null Int
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullInt(t, null, "null json")
+
+	var badType Int
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullInt(t, badType, "wrong type json")
+
+	var invalid Int
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullInt(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonIntegerNumber(t *testing.T) {
+	var i Int
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to int")
+	}
+}
+
+func TestTextUnmarshalInt(t *testing.T) {
+	var i Int
+	err := i.UnmarshalText([]byte("12345"))
+	maybePanic(err)
+	assertInt(t, i, "UnmarshalText() int")
+
+	var blank Int
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullInt(t, blank, "UnmarshalText() empty int")
+}
+
+func TestMarshalInt(t *testing.T) {
+	i := IntFrom(12345)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "12345", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalIntText(t *testing.T) {
+	i := IntFrom(12345)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "12345", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewInt(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestIntPointer(t *testing.T) {
+	i := IntFrom(12345)
+	ptr := i.Ptr()
+	if *ptr != 12345 {
+		t.Errorf("bad %s int: %#v ≠ %d\n", "pointer", ptr, 12345)
+	}
+
+	null := NewInt(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s int: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestIntIsNull(t *testing.T) {
+	i := IntFrom(12345)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewInt(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewInt(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestIntSetValid(t *testing.T) {
+	change := NewInt(0, false)
+	assertNullInt(t, change, "SetValid()")
+	change.SetValid(12345)
+	assertInt(t, change, "SetValid()")
+}
+
+func TestIntScan(t *testing.T) {
+	var i Int
+	err := i.Scan(12345)
+	maybePanic(err)
+	assertInt(t, i, "scanned int")
+
+	var null Int
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullInt(t, null, "scanned null")
+}
+
+func assertInt(t *testing.T, i Int, from string) {
+	if i.Int != 12345 {
+		t.Errorf("bad %s int: %d ≠ %d\n", from, i.Int, 12345)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullInt(t *testing.T, i Int, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/json.go

@@ -0,0 +1,157 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+	"fmt"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// JSON is a nullable []byte.
+type JSON struct {
+	JSON  []byte
+	Valid bool
+}
+
+// NewJSON creates a new JSON
+func NewJSON(b []byte, valid bool) JSON {
+	return JSON{
+		JSON:  b,
+		Valid: valid,
+	}
+}
+
+// JSONFrom creates a new JSON that will be invalid if nil.
+func JSONFrom(b []byte) JSON {
+	return NewJSON(b, b != nil)
+}
+
+// JSONFromPtr creates a new JSON that will be invalid if nil.
+func JSONFromPtr(b *[]byte) JSON {
+	if b == nil {
+		return NewJSON(nil, false)
+	}
+	n := NewJSON(*b, true)
+	return n
+}
+
+// Unmarshal will unmarshal your JSON stored in
+// your JSON object and store the result in the
+// value pointed to by dest.
+func (j JSON) Unmarshal(dest interface{}) error {
+	if dest == nil {
+		return errors.New("destination is nil, not a valid pointer to an object")
+	}
+
+	// Call our implementation of
+	// JSON MarshalJSON through json.Marshal
+	// to get the value of the JSON object
+	res, err := json.Marshal(j)
+	if err != nil {
+		return err
+	}
+
+	return json.Unmarshal(res, dest)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (j *JSON) UnmarshalJSON(data []byte) error {
+	if data == nil {
+		return fmt.Errorf("json: cannot unmarshal nil into Go value of type null.JSON")
+	}
+
+	if bytes.Equal(data, NullBytes) {
+		j.JSON = NullBytes
+		j.Valid = false
+		return nil
+	}
+
+	j.Valid = true
+	j.JSON = make([]byte, len(data))
+	copy(j.JSON, data)
+
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (j *JSON) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		j.JSON = nil
+		j.Valid = false
+	} else {
+		j.JSON = append(j.JSON[0:0], text...)
+		j.Valid = true
+	}
+
+	return nil
+}
+
+// Marshal will marshal the passed in object,
+// and store it in the JSON member on the JSON object.
+func (j *JSON) Marshal(obj interface{}) error {
+	res, err := json.Marshal(obj)
+	if err != nil {
+		return err
+	}
+
+	// Call our implementation of
+	// JSON UnmarshalJSON through json.Unmarshal
+	// to set the result to the JSON object
+	return json.Unmarshal(res, j)
+}
+
+// MarshalJSON implements json.Marshaler.
+func (j JSON) MarshalJSON() ([]byte, error) {
+	if len(j.JSON) == 0 || j.JSON == nil {
+		return NullBytes, nil
+	}
+	return j.JSON, nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (j JSON) MarshalText() ([]byte, error) {
+	if !j.Valid {
+		return nil, nil
+	}
+	return j.JSON, nil
+}
+
+// SetValid changes this JSON's value and also sets it to be non-null.
+func (j *JSON) SetValid(n []byte) {
+	j.JSON = n
+	j.Valid = true
+}
+
+// Ptr returns a pointer to this JSON's value, or a nil pointer if this JSON is null.
+func (j JSON) Ptr() *[]byte {
+	if !j.Valid {
+		return nil
+	}
+	return &j.JSON
+}
+
+// IsNull returns true for null or zero JSON's, for future omitempty support (Go 1.4?)
+func (j JSON) IsNull() bool {
+	return !j.Valid
+}
+
+// Scan implements the Scanner interface.
+func (j *JSON) Scan(value interface{}) error {
+	if value == nil {
+		j.JSON, j.Valid = []byte{}, false
+		return nil
+	}
+	j.Valid = true
+	return convert.ConvertAssign(&j.JSON, value)
+}
+
+// Value implements the driver Valuer interface.
+func (j JSON) Value() (driver.Value, error) {
+	if !j.Valid {
+		return nil, nil
+	}
+	return j.JSON, nil
+}

extras/null/json_test.go

@@ -0,0 +1,238 @@
+package null
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+)
+
+var (
+	jsonJSON = []byte(`"hello"`)
+)
+
+func TestJSONFrom(t *testing.T) {
+	i := JSONFrom([]byte(`"hello"`))
+	assertJSON(t, i, "JSONFrom()")
+
+	zero := JSONFrom(nil)
+	if zero.Valid {
+		t.Error("JSONFrom(nil)", "is valid, but should be invalid")
+	}
+
+	zero = JSONFrom([]byte{})
+	if !zero.Valid {
+		t.Error("JSONFrom([]byte{})", "is invalid, but should be valid")
+	}
+}
+
+func TestJSONFromPtr(t *testing.T) {
+	n := []byte(`"hello"`)
+	iptr := &n
+	i := JSONFromPtr(iptr)
+	assertJSON(t, i, "JSONFromPtr()")
+
+	null := JSONFromPtr(nil)
+	assertNullJSON(t, null, "JSONFromPtr(nil)")
+}
+
+type Test struct {
+	Name string
+	Age  int
+}
+
+func TestMarshal(t *testing.T) {
+	var i JSON
+
+	test := &Test{Name: "hello", Age: 15}
+
+	err := i.Marshal(test)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !bytes.Equal(i.JSON, []byte(`{"Name":"hello","Age":15}`)) {
+		t.Errorf("Mismatch between received and expected, got: %s", string(i.JSON))
+	}
+	if i.Valid == false {
+		t.Error("Expected valid true, got Valid false")
+	}
+
+	err = i.Marshal(nil)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !bytes.Equal(i.JSON, []byte("null")) {
+		t.Errorf("Expected null, but got %s", string(i.JSON))
+	}
+	if i.Valid == true {
+		t.Error("Expected Valid false, got Valid true")
+	}
+}
+
+func TestUnmarshal(t *testing.T) {
+	var i JSON
+
+	test := &Test{}
+
+	err := i.Unmarshal(test)
+	if err != nil {
+		t.Error(err)
+	}
+
+	x := &Test{Name: "hello", Age: 15}
+	err = i.Marshal(x)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if !bytes.Equal(i.JSON, []byte(`{"Name":"hello","Age":15}`)) {
+		t.Errorf("Mismatch between received and expected, got: %s", string(i.JSON))
+	}
+
+	err = i.Unmarshal(test)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if test.Age != 15 {
+		t.Errorf("Expected 15, got %d", test.Age)
+	}
+	if test.Name != "hello" {
+		t.Errorf("Expected name, got %s", test.Name)
+	}
+}
+
+func TestUnmarshalJSON(t *testing.T) {
+	var i JSON
+	err := json.Unmarshal(jsonJSON, &i)
+	maybePanic(err)
+	assertJSON(t, i, "[]byte json")
+
+	var ni JSON
+	err = ni.UnmarshalJSON([]byte{})
+	if ni.Valid == false {
+		t.Errorf("expected Valid to be true, got false")
+	}
+	if !bytes.Equal(ni.JSON, nil) {
+		t.Errorf("Expected JSON to be nil, but was not: %#v %#v", ni.JSON, []byte(nil))
+	}
+
+	var null JSON
+	err = null.UnmarshalJSON(nil)
+	if ni.Valid == false {
+		t.Errorf("expected Valid to be true, got false")
+	}
+	if !bytes.Equal(null.JSON, nil) {
+		t.Errorf("Expected JSON to be []byte nil, but was not: %#v %#v", null.JSON, []byte(nil))
+	}
+}
+
+func TestTextUnmarshalJSON(t *testing.T) {
+	var i JSON
+	err := i.UnmarshalText([]byte(`"hello"`))
+	maybePanic(err)
+	assertJSON(t, i, "UnmarshalText() []byte")
+
+	var blank JSON
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullJSON(t, blank, "UnmarshalText() empty []byte")
+}
+
+func TestMarshalJSON(t *testing.T) {
+	i := JSONFrom([]byte(`"hello"`))
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, `"hello"`, "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewJSON(nil, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalJSONText(t *testing.T) {
+	i := JSONFrom([]byte(`"hello"`))
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, `"hello"`, "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewJSON(nil, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestJSONPointer(t *testing.T) {
+	i := JSONFrom([]byte(`"hello"`))
+	ptr := i.Ptr()
+	if !bytes.Equal(*ptr, []byte(`"hello"`)) {
+		t.Errorf("bad %s []byte: %#v ≠ %s\n", "pointer", ptr, `"hello"`)
+	}
+
+	null := NewJSON(nil, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s []byte: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestJSONIsNull(t *testing.T) {
+	i := JSONFrom([]byte(`"hello"`))
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewJSON(nil, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewJSON(nil, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestJSONSetValid(t *testing.T) {
+	change := NewJSON(nil, false)
+	assertNullJSON(t, change, "SetValid()")
+	change.SetValid([]byte(`"hello"`))
+	assertJSON(t, change, "SetValid()")
+}
+
+func TestJSONScan(t *testing.T) {
+	var i JSON
+	err := i.Scan(`"hello"`)
+	maybePanic(err)
+	assertJSON(t, i, "scanned []byte")
+
+	var null JSON
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullJSON(t, null, "scanned null")
+}
+
+func assertJSON(t *testing.T, i JSON, from string) {
+	if !bytes.Equal(i.JSON, []byte(`"hello"`)) {
+		t.Errorf("bad %s []byte: %#v ≠ %#v\n", from, string(i.JSON), string([]byte(`"hello"`)))
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullJSON(t *testing.T, i JSON, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/nullable.go

@@ -0,0 +1,5 @@
+package null
+
+type Nullable interface {
+	IsNull() bool
+}

extras/null/string.go

@@ -0,0 +1,117 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// String is a nullable string. It supports SQL and JSON serialization.
+type String struct {
+	String string
+	Valid  bool
+}
+
+// StringFrom creates a new String that will never be blank.
+func StringFrom(s string) String {
+	return NewString(s, true)
+}
+
+// StringFromPtr creates a new String that be null if s is nil.
+func StringFromPtr(s *string) String {
+	if s == nil {
+		return NewString("", false)
+	}
+	return NewString(*s, true)
+}
+
+// NewString creates a new String
+func NewString(s string, valid bool) String {
+	return String{
+		String: s,
+		Valid:  valid,
+	}
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (s *String) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		s.String = ""
+		s.Valid = false
+		return nil
+	}
+
+	if err := json.Unmarshal(data, &s.String); err != nil {
+		return err
+	}
+
+	s.Valid = true
+	return nil
+}
+
+// MarshalJSON implements json.Marshaler.
+func (s String) MarshalJSON() ([]byte, error) {
+	if !s.Valid {
+		return NullBytes, nil
+	}
+	return json.Marshal(s.String)
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (s String) MarshalText() ([]byte, error) {
+	if !s.Valid {
+		return []byte{}, nil
+	}
+	return []byte(s.String), nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (s *String) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		s.Valid = false
+		return nil
+	}
+
+	s.String = string(text)
+	s.Valid = true
+	return nil
+}
+
+// SetValid changes this String's value and also sets it to be non-null.
+func (s *String) SetValid(v string) {
+	s.String = v
+	s.Valid = true
+}
+
+// Ptr returns a pointer to this String's value, or a nil pointer if this String is null.
+func (s String) Ptr() *string {
+	if !s.Valid {
+		return nil
+	}
+	return &s.String
+}
+
+// IsNull returns true for null strings, for potential future omitempty support.
+func (s String) IsNull() bool {
+	return !s.Valid
+}
+
+// Scan implements the Scanner interface.
+func (s *String) Scan(value interface{}) error {
+	if value == nil {
+		s.String, s.Valid = "", false
+		return nil
+	}
+	s.Valid = true
+	return convert.ConvertAssign(&s.String, value)
+}
+
+// Value implements the driver Valuer interface.
+func (s String) Value() (driver.Value, error) {
+	if !s.Valid {
+		return nil, nil
+	}
+	return s.String, nil
+}

extras/null/string_test.go

@@ -0,0 +1,206 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	stringJSON      = []byte(`"test"`)
+	blankStringJSON = []byte(`""`)
+
+	nullJSON    = []byte(`null`)
+	invalidJSON = []byte(`:)`)
+)
+
+type stringInStruct struct {
+	Test String `json:"test,omitempty"`
+}
+
+func TestStringFrom(t *testing.T) {
+	str := StringFrom("test")
+	assertStr(t, str, "StringFrom() string")
+
+	zero := StringFrom("")
+	if !zero.Valid {
+		t.Error("StringFrom(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestStringFromPtr(t *testing.T) {
+	s := "test"
+	sptr := &s
+	str := StringFromPtr(sptr)
+	assertStr(t, str, "StringFromPtr() string")
+
+	null := StringFromPtr(nil)
+	assertNullStr(t, null, "StringFromPtr(nil)")
+}
+
+func TestUnmarshalString(t *testing.T) {
+	var str String
+	err := json.Unmarshal(stringJSON, &str)
+	maybePanic(err)
+	assertStr(t, str, "string json")
+
+	var blank String
+	err = json.Unmarshal(blankStringJSON, &blank)
+	maybePanic(err)
+	if !blank.Valid {
+		t.Error("blank string should be valid")
+	}
+
+	var null String
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullStr(t, null, "null json")
+
+	var badType String
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullStr(t, badType, "wrong type json")
+
+	var invalid String
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullStr(t, invalid, "invalid json")
+}
+
+func TestTextUnmarshalString(t *testing.T) {
+	var str String
+	err := str.UnmarshalText([]byte("test"))
+	maybePanic(err)
+	assertStr(t, str, "UnmarshalText() string")
+
+	var null String
+	err = null.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullStr(t, null, "UnmarshalText() empty string")
+}
+
+func TestMarshalString(t *testing.T) {
+	str := StringFrom("test")
+	data, err := json.Marshal(str)
+	maybePanic(err)
+	assertJSONEquals(t, data, `"test"`, "non-empty json marshal")
+	data, err = str.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "test", "non-empty text marshal")
+
+	// empty values should be encoded as an empty string
+	zero := StringFrom("")
+	data, err = json.Marshal(zero)
+	maybePanic(err)
+	assertJSONEquals(t, data, `""`, "empty json marshal")
+	data, err = zero.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "string marshal text")
+
+	null := StringFromPtr(nil)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, `null`, "null json marshal")
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "string marshal text")
+}
+
+// Tests omitempty... broken until Go 1.4
+// func TestMarshalStringInStruct(t *testing.T) {
+// 	obj := stringInStruct{Test: StringFrom("")}
+// 	data, err := json.Marshal(obj)
+// 	maybePanic(err)
+// 	assertJSONEquals(t, data, `{}`, "null string in struct")
+// }
+
+func TestStringPointer(t *testing.T) {
+	str := StringFrom("test")
+	ptr := str.Ptr()
+	if *ptr != "test" {
+		t.Errorf("bad %s string: %#v ≠ %s\n", "pointer", ptr, "test")
+	}
+
+	null := NewString("", false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s string: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestStringIsNull(t *testing.T) {
+	str := StringFrom("test")
+	if str.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	blank := StringFrom("")
+	if blank.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	empty := NewString("", true)
+	if empty.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := StringFromPtr(nil)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	var testInt interface{}
+	testInt = empty
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestStringSetValid(t *testing.T) {
+	change := NewString("", false)
+	assertNullStr(t, change, "SetValid()")
+	change.SetValid("test")
+	assertStr(t, change, "SetValid()")
+}
+
+func TestStringScan(t *testing.T) {
+	var str String
+	err := str.Scan("test")
+	maybePanic(err)
+	assertStr(t, str, "scanned string")
+
+	var null String
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullStr(t, null, "scanned null")
+}
+
+func maybePanic(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+func assertStr(t *testing.T, s String, from string) {
+	if s.String != "test" {
+		t.Errorf("bad %s string: %s ≠ %s\n", from, s.String, "test")
+	}
+	if !s.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullStr(t *testing.T, s String, from string) {
+	if s.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}
+
+func assertJSONEquals(t *testing.T, data []byte, cmp string, from string) {
+	if string(data) != cmp {
+		t.Errorf("bad %s data: %s ≠ %s\n", from, data, cmp)
+	}
+}

extras/null/time.go

@@ -0,0 +1,123 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"fmt"
+	"time"
+)
+
+// Time is a nullable time.Time. It supports SQL and JSON serialization.
+type Time struct {
+	Time  time.Time
+	Valid bool
+}
+
+// NewTime creates a new Time.
+func NewTime(t time.Time, valid bool) Time {
+	return Time{
+		Time:  t,
+		Valid: valid,
+	}
+}
+
+// TimeFrom creates a new Time that will always be valid.
+func TimeFrom(t time.Time) Time {
+	return NewTime(t, true)
+}
+
+// TimeFromPtr creates a new Time that will be null if t is nil.
+func TimeFromPtr(t *time.Time) Time {
+	if t == nil {
+		return NewTime(time.Time{}, false)
+	}
+	return NewTime(*t, true)
+}
+
+// MarshalJSON implements json.Marshaler.
+func (t Time) MarshalJSON() ([]byte, error) {
+	if !t.Valid {
+		return NullBytes, nil
+	}
+	return t.Time.MarshalJSON()
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (t *Time) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		t.Valid = false
+		t.Time = time.Time{}
+		return nil
+	}
+
+	if err := t.Time.UnmarshalJSON(data); err != nil {
+		return err
+	}
+
+	t.Valid = true
+	return nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (t Time) MarshalText() ([]byte, error) {
+	if !t.Valid {
+		return NullBytes, nil
+	}
+	return t.Time.MarshalText()
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (t *Time) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		t.Valid = false
+		return nil
+	}
+	if err := t.Time.UnmarshalText(text); err != nil {
+		return err
+	}
+	t.Valid = true
+	return nil
+}
+
+// SetValid changes this Time's value and sets it to be non-null.
+func (t *Time) SetValid(v time.Time) {
+	t.Time = v
+	t.Valid = true
+}
+
+// Ptr returns a pointer to this Time's value, or a nil pointer if this Time is null.
+func (t Time) Ptr() *time.Time {
+	if !t.Valid {
+		return nil
+	}
+	return &t.Time
+}
+
+// IsNull returns true for invalid Times, for future omitempty support (Go 1.4?)
+func (t Time) IsNull() bool {
+	return !t.Valid
+}
+
+// Scan implements the Scanner interface.
+func (t *Time) Scan(value interface{}) error {
+	var err error
+	switch x := value.(type) {
+	case time.Time:
+		t.Time = x
+	case nil:
+		t.Valid = false
+		return nil
+	default:
+		err = fmt.Errorf("null: cannot scan type %T into null.Time: %v", value, value)
+	}
+	t.Valid = err == nil
+	return err
+}
+
+// Value implements the driver Valuer interface.
+func (t Time) Value() (driver.Value, error) {
+	if !t.Valid {
+		return nil, nil
+	}
+	return t.Time, nil
+}

extras/null/time_test.go

@@ -0,0 +1,178 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+	"time"
+)
+
+var (
+	timeString   = "2012-12-21T21:21:21Z"
+	timeJSON     = []byte(`"` + timeString + `"`)
+	nullTimeJSON = []byte(`null`)
+	timeValue, _ = time.Parse(time.RFC3339, timeString)
+	badObject    = []byte(`{"hello": "world"}`)
+)
+
+func TestUnmarshalTimeJSON(t *testing.T) {
+	var ti Time
+	err := json.Unmarshal(timeJSON, &ti)
+	maybePanic(err)
+	assertTime(t, ti, "UnmarshalJSON() json")
+
+	var null Time
+	err = json.Unmarshal(nullTimeJSON, &null)
+	maybePanic(err)
+	assertNullTime(t, null, "null time json")
+
+	var invalid Time
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*time.ParseError); !ok {
+		t.Errorf("expected json.ParseError, not %T", err)
+	}
+	assertNullTime(t, invalid, "invalid from object json")
+
+	var bad Time
+	err = json.Unmarshal(badObject, &bad)
+	if err == nil {
+		t.Errorf("expected error: bad object")
+	}
+	assertNullTime(t, bad, "bad from object json")
+
+	var wrongType Time
+	err = json.Unmarshal(intJSON, &wrongType)
+	if err == nil {
+		t.Errorf("expected error: wrong type JSON")
+	}
+	assertNullTime(t, wrongType, "wrong type object json")
+}
+
+func TestUnmarshalTimeText(t *testing.T) {
+	ti := TimeFrom(timeValue)
+	txt, err := ti.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, txt, timeString, "marshal text")
+
+	var unmarshal Time
+	err = unmarshal.UnmarshalText(txt)
+	maybePanic(err)
+	assertTime(t, unmarshal, "unmarshal text")
+
+	var invalid Time
+	err = invalid.UnmarshalText([]byte("hello world"))
+	if err == nil {
+		t.Error("expected error")
+	}
+	assertNullTime(t, invalid, "bad string")
+}
+
+func TestMarshalTime(t *testing.T) {
+	ti := TimeFrom(timeValue)
+	data, err := json.Marshal(ti)
+	maybePanic(err)
+	assertJSONEquals(t, data, string(timeJSON), "non-empty json marshal")
+
+	ti.Valid = false
+	data, err = json.Marshal(ti)
+	maybePanic(err)
+	assertJSONEquals(t, data, string(nullJSON), "null json marshal")
+}
+
+func TestTimeFrom(t *testing.T) {
+	ti := TimeFrom(timeValue)
+	assertTime(t, ti, "TimeFrom() time.Time")
+}
+
+func TestTimeFromPtr(t *testing.T) {
+	ti := TimeFromPtr(&timeValue)
+	assertTime(t, ti, "TimeFromPtr() time")
+
+	null := TimeFromPtr(nil)
+	assertNullTime(t, null, "TimeFromPtr(nil)")
+}
+
+func TestTimeSetValid(t *testing.T) {
+	var ti time.Time
+	change := NewTime(ti, false)
+	assertNullTime(t, change, "SetValid()")
+	change.SetValid(timeValue)
+	assertTime(t, change, "SetValid()")
+}
+
+func TestTimeIsNull(t *testing.T) {
+	ti := TimeFrom(time.Now())
+	if ti.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewTime(time.Now(), false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewTime(time.Time{}, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestTimePointer(t *testing.T) {
+	ti := TimeFrom(timeValue)
+	ptr := ti.Ptr()
+	if *ptr != timeValue {
+		t.Errorf("bad %s time: %#v ≠ %v\n", "pointer", ptr, timeValue)
+	}
+
+	var nt time.Time
+	null := NewTime(nt, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s time: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestTimeScanValue(t *testing.T) {
+	var ti Time
+	err := ti.Scan(timeValue)
+	maybePanic(err)
+	assertTime(t, ti, "scanned time")
+	if v, err := ti.Value(); v != timeValue || err != nil {
+		t.Error("bad value or err:", v, err)
+	}
+
+	var null Time
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullTime(t, null, "scanned null")
+	if v, err := null.Value(); v != nil || err != nil {
+		t.Error("bad value or err:", v, err)
+	}
+
+	var wrong Time
+	err = wrong.Scan(int64(42))
+	if err == nil {
+		t.Error("expected error")
+	}
+	assertNullTime(t, wrong, "scanned wrong")
+}
+
+func assertTime(t *testing.T, ti Time, from string) {
+	if ti.Time != timeValue {
+		t.Errorf("bad %v time: %v ≠ %v\n", from, ti.Time, timeValue)
+	}
+	if !ti.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullTime(t *testing.T, ti Time, from string) {
+	if ti.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/uint.go

@@ -0,0 +1,123 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Uint is an nullable uint.
+type Uint struct {
+	Uint  uint
+	Valid bool
+}
+
+// NewUint creates a new Uint
+func NewUint(i uint, valid bool) Uint {
+	return Uint{
+		Uint:  i,
+		Valid: valid,
+	}
+}
+
+// UintFrom creates a new Uint that will always be valid.
+func UintFrom(i uint) Uint {
+	return NewUint(i, true)
+}
+
+// UintFromPtr creates a new Uint that be null if i is nil.
+func UintFromPtr(i *uint) Uint {
+	if i == nil {
+		return NewUint(0, false)
+	}
+	return NewUint(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (u *Uint) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		u.Valid = false
+		u.Uint = 0
+		return nil
+	}
+
+	var x uint64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	u.Uint = uint(x)
+	u.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (u *Uint) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		u.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseUint(string(text), 10, 0)
+	u.Valid = err == nil
+	if u.Valid {
+		u.Uint = uint(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (u Uint) MarshalJSON() ([]byte, error) {
+	if !u.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (u Uint) MarshalText() ([]byte, error) {
+	if !u.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint), 10)), nil
+}
+
+// SetValid changes this Uint's value and also sets it to be non-null.
+func (u *Uint) SetValid(n uint) {
+	u.Uint = n
+	u.Valid = true
+}
+
+// Ptr returns a pointer to this Uint's value, or a nil pointer if this Uint is null.
+func (u Uint) Ptr() *uint {
+	if !u.Valid {
+		return nil
+	}
+	return &u.Uint
+}
+
+// IsNull returns true for invalid Uints, for future omitempty support (Go 1.4?)
+func (u Uint) IsNull() bool {
+	return !u.Valid
+}
+
+// Scan implements the Scanner interface.
+func (u *Uint) Scan(value interface{}) error {
+	if value == nil {
+		u.Uint, u.Valid = 0, false
+		return nil
+	}
+	u.Valid = true
+	return convert.ConvertAssign(&u.Uint, value)
+}
+
+// Value implements the driver Valuer interface.
+func (u Uint) Value() (driver.Value, error) {
+	if !u.Valid {
+		return nil, nil
+	}
+	return int64(u.Uint), nil
+}

extras/null/uint16.go

@@ -0,0 +1,129 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Uint16 is an nullable uint16.
+type Uint16 struct {
+	Uint16 uint16
+	Valid  bool
+}
+
+// NewUint16 creates a new Uint16
+func NewUint16(i uint16, valid bool) Uint16 {
+	return Uint16{
+		Uint16: i,
+		Valid:  valid,
+	}
+}
+
+// Uint16From creates a new Uint16 that will always be valid.
+func Uint16From(i uint16) Uint16 {
+	return NewUint16(i, true)
+}
+
+// Uint16FromPtr creates a new Uint16 that be null if i is nil.
+func Uint16FromPtr(i *uint16) Uint16 {
+	if i == nil {
+		return NewUint16(0, false)
+	}
+	return NewUint16(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (u *Uint16) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		u.Valid = false
+		u.Uint16 = 0
+		return nil
+	}
+
+	var x uint64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if x > math.MaxUint16 {
+		return fmt.Errorf("json: %d overflows max uint8 value", x)
+	}
+
+	u.Uint16 = uint16(x)
+	u.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (u *Uint16) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		u.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseUint(string(text), 10, 16)
+	u.Valid = err == nil
+	if u.Valid {
+		u.Uint16 = uint16(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (u Uint16) MarshalJSON() ([]byte, error) {
+	if !u.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint16), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (u Uint16) MarshalText() ([]byte, error) {
+	if !u.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint16), 10)), nil
+}
+
+// SetValid changes this Uint16's value and also sets it to be non-null.
+func (u *Uint16) SetValid(n uint16) {
+	u.Uint16 = n
+	u.Valid = true
+}
+
+// Ptr returns a pointer to this Uint16's value, or a nil pointer if this Uint16 is null.
+func (u Uint16) Ptr() *uint16 {
+	if !u.Valid {
+		return nil
+	}
+	return &u.Uint16
+}
+
+// IsNull returns true for invalid Uint16's, for future omitempty support (Go 1.4?)
+func (u Uint16) IsNull() bool {
+	return !u.Valid
+}
+
+// Scan implements the Scanner interface.
+func (u *Uint16) Scan(value interface{}) error {
+	if value == nil {
+		u.Uint16, u.Valid = 0, false
+		return nil
+	}
+	u.Valid = true
+	return convert.ConvertAssign(&u.Uint16, value)
+}
+
+// Value implements the driver Valuer interface.
+func (u Uint16) Value() (driver.Value, error) {
+	if !u.Valid {
+		return nil, nil
+	}
+	return int64(u.Uint16), nil
+}

extras/null/uint16_test.go

@@ -0,0 +1,191 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	uint16JSON = []byte(`65534`)
+)
+
+func TestUint16From(t *testing.T) {
+	i := Uint16From(65534)
+	assertUint16(t, i, "Uint16From()")
+
+	zero := Uint16From(0)
+	if !zero.Valid {
+		t.Error("Uint16From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestUint16FromPtr(t *testing.T) {
+	n := uint16(65534)
+	iptr := &n
+	i := Uint16FromPtr(iptr)
+	assertUint16(t, i, "Uint16FromPtr()")
+
+	null := Uint16FromPtr(nil)
+	assertNullUint16(t, null, "Uint16FromPtr(nil)")
+}
+
+func TestUnmarshalUint16(t *testing.T) {
+	var i Uint16
+	err := json.Unmarshal(uint16JSON, &i)
+	maybePanic(err)
+	assertUint16(t, i, "uint16 json")
+
+	var null Uint16
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullUint16(t, null, "null json")
+
+	var badType Uint16
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullUint16(t, badType, "wrong type json")
+
+	var invalid Uint16
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullUint16(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonUintegerNumber16(t *testing.T) {
+	var i Uint16
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to uint16")
+	}
+}
+
+func TestUnmarshalUint16Overflow(t *testing.T) {
+	uint16Overflow := int64(math.MaxUint16)
+
+	// Max uint16 should decode successfully
+	var i Uint16
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(uint16Overflow), 10)), &i)
+	maybePanic(err)
+
+	// Attempt to overflow
+	uint16Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(uint16Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows uint16")
+	}
+}
+
+func TestTextUnmarshalUint16(t *testing.T) {
+	var i Uint16
+	err := i.UnmarshalText([]byte("65534"))
+	maybePanic(err)
+	assertUint16(t, i, "UnmarshalText() uint16")
+
+	var blank Uint16
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullUint16(t, blank, "UnmarshalText() empty uint16")
+}
+
+func TestMarshalUint16(t *testing.T) {
+	i := Uint16From(65534)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "65534", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint16(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalUint16Text(t *testing.T) {
+	i := Uint16From(65534)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "65534", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint16(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestUint16Pointer(t *testing.T) {
+	i := Uint16From(65534)
+	ptr := i.Ptr()
+	if *ptr != 65534 {
+		t.Errorf("bad %s uint16: %#v ≠ %d\n", "pointer", ptr, 65534)
+	}
+
+	null := NewUint16(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s uint16: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestUint16IsNull(t *testing.T) {
+	i := Uint16From(65534)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewUint16(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewUint16(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestUint16SetValid(t *testing.T) {
+	change := NewUint16(0, false)
+	assertNullUint16(t, change, "SetValid()")
+	change.SetValid(65534)
+	assertUint16(t, change, "SetValid()")
+}
+
+func TestUint16Scan(t *testing.T) {
+	var i Uint16
+	err := i.Scan(65534)
+	maybePanic(err)
+	assertUint16(t, i, "scanned uint16")
+
+	var null Uint16
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullUint16(t, null, "scanned null")
+}
+
+func assertUint16(t *testing.T, i Uint16, from string) {
+	if i.Uint16 != 65534 {
+		t.Errorf("bad %s uint16: %d ≠ %d\n", from, i.Uint16, 65534)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullUint16(t *testing.T, i Uint16, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/uint32.go

@@ -0,0 +1,129 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Uint32 is an nullable uint32.
+type Uint32 struct {
+	Uint32 uint32
+	Valid  bool
+}
+
+// NewUint32 creates a new Uint32
+func NewUint32(i uint32, valid bool) Uint32 {
+	return Uint32{
+		Uint32: i,
+		Valid:  valid,
+	}
+}
+
+// Uint32From creates a new Uint32 that will always be valid.
+func Uint32From(i uint32) Uint32 {
+	return NewUint32(i, true)
+}
+
+// Uint32FromPtr creates a new Uint32 that be null if i is nil.
+func Uint32FromPtr(i *uint32) Uint32 {
+	if i == nil {
+		return NewUint32(0, false)
+	}
+	return NewUint32(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (u *Uint32) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		u.Valid = false
+		u.Uint32 = 0
+		return nil
+	}
+
+	var x uint64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if x > math.MaxUint32 {
+		return fmt.Errorf("json: %d overflows max uint32 value", x)
+	}
+
+	u.Uint32 = uint32(x)
+	u.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (u *Uint32) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		u.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseUint(string(text), 10, 32)
+	u.Valid = err == nil
+	if u.Valid {
+		u.Uint32 = uint32(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (u Uint32) MarshalJSON() ([]byte, error) {
+	if !u.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint32), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (u Uint32) MarshalText() ([]byte, error) {
+	if !u.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint32), 10)), nil
+}
+
+// SetValid changes this Uint32's value and also sets it to be non-null.
+func (u *Uint32) SetValid(n uint32) {
+	u.Uint32 = n
+	u.Valid = true
+}
+
+// Ptr returns a pointer to this Uint32's value, or a nil pointer if this Uint32 is null.
+func (u Uint32) Ptr() *uint32 {
+	if !u.Valid {
+		return nil
+	}
+	return &u.Uint32
+}
+
+// IsNull returns true for invalid Uint32's, for future omitempty support (Go 1.4?)
+func (u Uint32) IsNull() bool {
+	return !u.Valid
+}
+
+// Scan implements the Scanner interface.
+func (u *Uint32) Scan(value interface{}) error {
+	if value == nil {
+		u.Uint32, u.Valid = 0, false
+		return nil
+	}
+	u.Valid = true
+	return convert.ConvertAssign(&u.Uint32, value)
+}
+
+// Value implements the driver Valuer interface.
+func (u Uint32) Value() (driver.Value, error) {
+	if !u.Valid {
+		return nil, nil
+	}
+	return uint64(u.Uint32), nil
+}

extras/null/uint32_test.go

@@ -0,0 +1,191 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	uint32JSON = []byte(`4294967294`)
+)
+
+func TestUint32From(t *testing.T) {
+	i := Uint32From(4294967294)
+	assertUint32(t, i, "Uint32From()")
+
+	zero := Uint32From(0)
+	if !zero.Valid {
+		t.Error("Uint32From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestUint32FromPtr(t *testing.T) {
+	n := uint32(4294967294)
+	iptr := &n
+	i := Uint32FromPtr(iptr)
+	assertUint32(t, i, "Uint32FromPtr()")
+
+	null := Uint32FromPtr(nil)
+	assertNullUint32(t, null, "Uint32FromPtr(nil)")
+}
+
+func TestUnmarshalUint32(t *testing.T) {
+	var i Uint32
+	err := json.Unmarshal(uint32JSON, &i)
+	maybePanic(err)
+	assertUint32(t, i, "uint32 json")
+
+	var null Uint32
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullUint32(t, null, "null json")
+
+	var badType Uint32
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullUint32(t, badType, "wrong type json")
+
+	var invalid Uint32
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullUint32(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonUintegerNumber32(t *testing.T) {
+	var i Uint32
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to uint32")
+	}
+}
+
+func TestUnmarshalUint32Overflow(t *testing.T) {
+	uint32Overflow := int64(math.MaxUint32)
+
+	// Max uint32 should decode successfully
+	var i Uint32
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(uint32Overflow), 10)), &i)
+	maybePanic(err)
+
+	// Attempt to overflow
+	uint32Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(uint32Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows uint32")
+	}
+}
+
+func TestTextUnmarshalUint32(t *testing.T) {
+	var i Uint32
+	err := i.UnmarshalText([]byte("4294967294"))
+	maybePanic(err)
+	assertUint32(t, i, "UnmarshalText() uint32")
+
+	var blank Uint32
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullUint32(t, blank, "UnmarshalText() empty uint32")
+}
+
+func TestMarshalUint32(t *testing.T) {
+	i := Uint32From(4294967294)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "4294967294", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint32(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalUint32Text(t *testing.T) {
+	i := Uint32From(4294967294)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "4294967294", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint32(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestUint32Pointer(t *testing.T) {
+	i := Uint32From(4294967294)
+	ptr := i.Ptr()
+	if *ptr != 4294967294 {
+		t.Errorf("bad %s uint32: %#v ≠ %d\n", "pointer", ptr, 4294967294)
+	}
+
+	null := NewUint32(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s uint32: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestUint32IsNull(t *testing.T) {
+	i := Uint32From(4294967294)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewUint32(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewUint32(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestUint32SetValid(t *testing.T) {
+	change := NewUint32(0, false)
+	assertNullUint32(t, change, "SetValid()")
+	change.SetValid(4294967294)
+	assertUint32(t, change, "SetValid()")
+}
+
+func TestUint32Scan(t *testing.T) {
+	var i Uint32
+	err := i.Scan(4294967294)
+	maybePanic(err)
+	assertUint32(t, i, "scanned uint32")
+
+	var null Uint32
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullUint32(t, null, "scanned null")
+}
+
+func assertUint32(t *testing.T, i Uint32, from string) {
+	if i.Uint32 != 4294967294 {
+		t.Errorf("bad %s uint32: %d ≠ %d\n", from, i.Uint32, 4294967294)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullUint32(t *testing.T, i Uint32, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/uint64.go

@@ -0,0 +1,121 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Uint64 is an nullable uint64.
+type Uint64 struct {
+	Uint64 uint64
+	Valid  bool
+}
+
+// NewUint64 creates a new Uint64
+func NewUint64(i uint64, valid bool) Uint64 {
+	return Uint64{
+		Uint64: i,
+		Valid:  valid,
+	}
+}
+
+// Uint64From creates a new Uint64 that will always be valid.
+func Uint64From(i uint64) Uint64 {
+	return NewUint64(i, true)
+}
+
+// Uint64FromPtr creates a new Uint64 that be null if i is nil.
+func Uint64FromPtr(i *uint64) Uint64 {
+	if i == nil {
+		return NewUint64(0, false)
+	}
+	return NewUint64(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (u *Uint64) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		u.Uint64 = 0
+		u.Valid = false
+		return nil
+	}
+
+	if err := json.Unmarshal(data, &u.Uint64); err != nil {
+		return err
+	}
+
+	u.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (u *Uint64) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		u.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseUint(string(text), 10, 64)
+	u.Valid = err == nil
+	if u.Valid {
+		u.Uint64 = uint64(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (u Uint64) MarshalJSON() ([]byte, error) {
+	if !u.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatUint(u.Uint64, 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (u Uint64) MarshalText() ([]byte, error) {
+	if !u.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatUint(u.Uint64, 10)), nil
+}
+
+// SetValid changes this Uint64's value and also sets it to be non-null.
+func (u *Uint64) SetValid(n uint64) {
+	u.Uint64 = n
+	u.Valid = true
+}
+
+// Ptr returns a pointer to this Uint64's value, or a nil pointer if this Uint64 is null.
+func (u Uint64) Ptr() *uint64 {
+	if !u.Valid {
+		return nil
+	}
+	return &u.Uint64
+}
+
+// IsNull returns true for invalid Uint64's, for future omitempty support (Go 1.4?)
+func (u Uint64) IsNull() bool {
+	return !u.Valid
+}
+
+// Scan implements the Scanner interface.
+func (u *Uint64) Scan(value interface{}) error {
+	if value == nil {
+		u.Uint64, u.Valid = 0, false
+		return nil
+	}
+	u.Valid = true
+	return convert.ConvertAssign(&u.Uint64, value)
+}
+
+// Value implements the driver Valuer interface.
+func (u Uint64) Value() (driver.Value, error) {
+	if !u.Valid {
+		return nil, nil
+	}
+	return int64(u.Uint64), nil
+}

extras/null/uint64_test.go

@@ -0,0 +1,173 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	uint64JSON = []byte(`18446744073709551614`)
+)
+
+func TestUint64From(t *testing.T) {
+	i := Uint64From(18446744073709551614)
+	assertUint64(t, i, "Uint64From()")
+
+	zero := Uint64From(0)
+	if !zero.Valid {
+		t.Error("Uint64From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestUint64FromPtr(t *testing.T) {
+	n := uint64(18446744073709551614)
+	iptr := &n
+	i := Uint64FromPtr(iptr)
+	assertUint64(t, i, "Uint64FromPtr()")
+
+	null := Uint64FromPtr(nil)
+	assertNullUint64(t, null, "Uint64FromPtr(nil)")
+}
+
+func TestUnmarshalUint64(t *testing.T) {
+	var i Uint64
+	err := json.Unmarshal(uint64JSON, &i)
+	maybePanic(err)
+	assertUint64(t, i, "uint64 json")
+
+	var null Uint64
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullUint64(t, null, "null json")
+
+	var badType Uint64
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullUint64(t, badType, "wrong type json")
+
+	var invalid Uint64
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullUint64(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonUintegerNumber64(t *testing.T) {
+	var i Uint64
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to uint64")
+	}
+}
+
+func TestTextUnmarshalUint64(t *testing.T) {
+	var i Uint64
+	err := i.UnmarshalText([]byte("18446744073709551614"))
+	maybePanic(err)
+	assertUint64(t, i, "UnmarshalText() uint64")
+
+	var blank Uint64
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullUint64(t, blank, "UnmarshalText() empty uint64")
+}
+
+func TestMarshalUint64(t *testing.T) {
+	i := Uint64From(18446744073709551614)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "18446744073709551614", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint64(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalUint64Text(t *testing.T) {
+	i := Uint64From(18446744073709551614)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "18446744073709551614", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint64(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestUint64Pointer(t *testing.T) {
+	i := Uint64From(18446744073709551614)
+	ptr := i.Ptr()
+	if *ptr != 18446744073709551614 {
+		t.Errorf("bad %s uint64: %#v ≠ %d\n", "pointer", ptr, uint64(18446744073709551614))
+	}
+
+	null := NewUint64(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s uint64: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestUint64IsNull(t *testing.T) {
+	i := Uint64From(18446744073709551614)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewUint64(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewUint64(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestUint64SetValid(t *testing.T) {
+	change := NewUint64(0, false)
+	assertNullUint64(t, change, "SetValid()")
+	change.SetValid(18446744073709551614)
+	assertUint64(t, change, "SetValid()")
+}
+
+func TestUint64Scan(t *testing.T) {
+	var i Uint64
+	err := i.Scan(uint64(18446744073709551614))
+	maybePanic(err)
+	assertUint64(t, i, "scanned uint64")
+
+	var null Uint64
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullUint64(t, null, "scanned null")
+}
+
+func assertUint64(t *testing.T, i Uint64, from string) {
+	if i.Uint64 != 18446744073709551614 {
+		t.Errorf("bad %s uint64: %d ≠ %d\n", from, i.Uint64, uint64(18446744073709551614))
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullUint64(t *testing.T, i Uint64, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/uint8.go

@@ -0,0 +1,129 @@
+package null
+
+import (
+	"bytes"
+	"database/sql/driver"
+	"encoding/json"
+	"fmt"
+	"math"
+	"strconv"
+
+	"gopkg.in/nullbio/null.v6/convert"
+)
+
+// Uint8 is an nullable uint8.
+type Uint8 struct {
+	Uint8 uint8
+	Valid bool
+}
+
+// NewUint8 creates a new Uint8
+func NewUint8(i uint8, valid bool) Uint8 {
+	return Uint8{
+		Uint8: i,
+		Valid: valid,
+	}
+}
+
+// Uint8From creates a new Uint8 that will always be valid.
+func Uint8From(i uint8) Uint8 {
+	return NewUint8(i, true)
+}
+
+// Uint8FromPtr creates a new Uint8 that be null if i is nil.
+func Uint8FromPtr(i *uint8) Uint8 {
+	if i == nil {
+		return NewUint8(0, false)
+	}
+	return NewUint8(*i, true)
+}
+
+// UnmarshalJSON implements json.Unmarshaler.
+func (u *Uint8) UnmarshalJSON(data []byte) error {
+	if bytes.Equal(data, NullBytes) {
+		u.Valid = false
+		u.Uint8 = 0
+		return nil
+	}
+
+	var x uint64
+	if err := json.Unmarshal(data, &x); err != nil {
+		return err
+	}
+
+	if x > math.MaxUint8 {
+		return fmt.Errorf("json: %d overflows max uint8 value", x)
+	}
+
+	u.Uint8 = uint8(x)
+	u.Valid = true
+	return nil
+}
+
+// UnmarshalText implements encoding.TextUnmarshaler.
+func (u *Uint8) UnmarshalText(text []byte) error {
+	if text == nil || len(text) == 0 {
+		u.Valid = false
+		return nil
+	}
+	var err error
+	res, err := strconv.ParseUint(string(text), 10, 8)
+	u.Valid = err == nil
+	if u.Valid {
+		u.Uint8 = uint8(res)
+	}
+	return err
+}
+
+// MarshalJSON implements json.Marshaler.
+func (u Uint8) MarshalJSON() ([]byte, error) {
+	if !u.Valid {
+		return NullBytes, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint8), 10)), nil
+}
+
+// MarshalText implements encoding.TextMarshaler.
+func (u Uint8) MarshalText() ([]byte, error) {
+	if !u.Valid {
+		return []byte{}, nil
+	}
+	return []byte(strconv.FormatUint(uint64(u.Uint8), 10)), nil
+}
+
+// SetValid changes this Uint8's value and also sets it to be non-null.
+func (u *Uint8) SetValid(n uint8) {
+	u.Uint8 = n
+	u.Valid = true
+}
+
+// Ptr returns a pointer to this Uint8's value, or a nil pointer if this Uint8 is null.
+func (u Uint8) Ptr() *uint8 {
+	if !u.Valid {
+		return nil
+	}
+	return &u.Uint8
+}
+
+// IsNull returns true for invalid Uint8's, for future omitempty support (Go 1.4?)
+func (u Uint8) IsNull() bool {
+	return !u.Valid
+}
+
+// Scan implements the Scanner interface.
+func (u *Uint8) Scan(value interface{}) error {
+	if value == nil {
+		u.Uint8, u.Valid = 0, false
+		return nil
+	}
+	u.Valid = true
+	return convert.ConvertAssign(&u.Uint8, value)
+}
+
+// Value implements the driver Valuer interface.
+func (u Uint8) Value() (driver.Value, error) {
+	if !u.Valid {
+		return nil, nil
+	}
+	return int64(u.Uint8), nil
+}

extras/null/uint8_test.go

@@ -0,0 +1,191 @@
+package null
+
+import (
+	"encoding/json"
+	"math"
+	"strconv"
+	"testing"
+)
+
+var (
+	uint8JSON = []byte(`254`)
+)
+
+func TestUint8From(t *testing.T) {
+	i := Uint8From(254)
+	assertUint8(t, i, "Uint8From()")
+
+	zero := Uint8From(0)
+	if !zero.Valid {
+		t.Error("Uint8From(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestUint8FromPtr(t *testing.T) {
+	n := uint8(254)
+	iptr := &n
+	i := Uint8FromPtr(iptr)
+	assertUint8(t, i, "Uint8FromPtr()")
+
+	null := Uint8FromPtr(nil)
+	assertNullUint8(t, null, "Uint8FromPtr(nil)")
+}
+
+func TestUnmarshalUint8(t *testing.T) {
+	var i Uint8
+	err := json.Unmarshal(uint8JSON, &i)
+	maybePanic(err)
+	assertUint8(t, i, "uint8 json")
+
+	var null Uint8
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullUint8(t, null, "null json")
+
+	var badType Uint8
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullUint8(t, badType, "wrong type json")
+
+	var invalid Uint8
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullUint8(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonUintegerNumber8(t *testing.T) {
+	var i Uint8
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-integer number coerced to uint8")
+	}
+}
+
+func TestUnmarshalUint8Overflow(t *testing.T) {
+	uint8Overflow := int64(math.MaxUint8)
+
+	// Max uint8 should decode successfully
+	var i Uint8
+	err := json.Unmarshal([]byte(strconv.FormatUint(uint64(uint8Overflow), 10)), &i)
+	maybePanic(err)
+
+	// Attempt to overflow
+	uint8Overflow++
+	err = json.Unmarshal([]byte(strconv.FormatUint(uint64(uint8Overflow), 10)), &i)
+	if err == nil {
+		panic("err should be present; decoded value overflows uint8")
+	}
+}
+
+func TestTextUnmarshalUint8(t *testing.T) {
+	var i Uint8
+	err := i.UnmarshalText([]byte("254"))
+	maybePanic(err)
+	assertUint8(t, i, "UnmarshalText() uint8")
+
+	var blank Uint8
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullUint8(t, blank, "UnmarshalText() empty uint8")
+}
+
+func TestMarshalUint8(t *testing.T) {
+	i := Uint8From(254)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "254", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint8(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalUint8Text(t *testing.T) {
+	i := Uint8From(254)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "254", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint8(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestUint8Pointer(t *testing.T) {
+	i := Uint8From(254)
+	ptr := i.Ptr()
+	if *ptr != 254 {
+		t.Errorf("bad %s uint8: %#v ≠ %d\n", "pointer", ptr, 254)
+	}
+
+	null := NewUint8(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s uint8: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestUint8IsNull(t *testing.T) {
+	i := Uint8From(254)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewUint8(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewUint8(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestUint8SetValid(t *testing.T) {
+	change := NewUint8(0, false)
+	assertNullUint8(t, change, "SetValid()")
+	change.SetValid(254)
+	assertUint8(t, change, "SetValid()")
+}
+
+func TestUint8Scan(t *testing.T) {
+	var i Uint8
+	err := i.Scan(254)
+	maybePanic(err)
+	assertUint8(t, i, "scanned uint8")
+
+	var null Uint8
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullUint8(t, null, "scanned null")
+}
+
+func assertUint8(t *testing.T, i Uint8, from string) {
+	if i.Uint8 != 254 {
+		t.Errorf("bad %s uint8: %d ≠ %d\n", from, i.Uint8, 254)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullUint8(t *testing.T, i Uint8, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/null/uint_test.go

@@ -0,0 +1,173 @@
+package null
+
+import (
+	"encoding/json"
+	"testing"
+)
+
+var (
+	uintJSON = []byte(`12345`)
+)
+
+func TestUintFrom(t *testing.T) {
+	i := UintFrom(12345)
+	assertUint(t, i, "UintFrom()")
+
+	zero := UintFrom(0)
+	if !zero.Valid {
+		t.Error("UintFrom(0)", "is invalid, but should be valid")
+	}
+}
+
+func TestUintFromPtr(t *testing.T) {
+	n := uint(12345)
+	iptr := &n
+	i := UintFromPtr(iptr)
+	assertUint(t, i, "UintFromPtr()")
+
+	null := UintFromPtr(nil)
+	assertNullUint(t, null, "UintFromPtr(nil)")
+}
+
+func TestUnmarshalUint(t *testing.T) {
+	var i Uint
+	err := json.Unmarshal(uintJSON, &i)
+	maybePanic(err)
+	assertUint(t, i, "uint json")
+
+	var null Uint
+	err = json.Unmarshal(nullJSON, &null)
+	maybePanic(err)
+	assertNullUint(t, null, "null json")
+
+	var badType Uint
+	err = json.Unmarshal(boolJSON, &badType)
+	if err == nil {
+		panic("err should not be nil")
+	}
+	assertNullUint(t, badType, "wrong type json")
+
+	var invalid Uint
+	err = invalid.UnmarshalJSON(invalidJSON)
+	if _, ok := err.(*json.SyntaxError); !ok {
+		t.Errorf("expected json.SyntaxError, not %T", err)
+	}
+	assertNullUint(t, invalid, "invalid json")
+}
+
+func TestUnmarshalNonUintegerNumber(t *testing.T) {
+	var i Uint
+	err := json.Unmarshal(float64JSON, &i)
+	if err == nil {
+		panic("err should be present; non-uinteger number coerced to uint")
+	}
+}
+
+func TestTextUnmarshalUint(t *testing.T) {
+	var i Uint
+	err := i.UnmarshalText([]byte("12345"))
+	maybePanic(err)
+	assertUint(t, i, "UnmarshalText() uint")
+
+	var blank Uint
+	err = blank.UnmarshalText([]byte(""))
+	maybePanic(err)
+	assertNullUint(t, blank, "UnmarshalText() empty uint")
+}
+
+func TestMarshalUint(t *testing.T) {
+	i := UintFrom(12345)
+	data, err := json.Marshal(i)
+	maybePanic(err)
+	assertJSONEquals(t, data, "12345", "non-empty json marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint(0, false)
+	data, err = json.Marshal(null)
+	maybePanic(err)
+	assertJSONEquals(t, data, "null", "null json marshal")
+}
+
+func TestMarshalUintText(t *testing.T) {
+	i := UintFrom(12345)
+	data, err := i.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "12345", "non-empty text marshal")
+
+	// invalid values should be encoded as null
+	null := NewUint(0, false)
+	data, err = null.MarshalText()
+	maybePanic(err)
+	assertJSONEquals(t, data, "", "null text marshal")
+}
+
+func TestUintPointer(t *testing.T) {
+	i := UintFrom(12345)
+	ptr := i.Ptr()
+	if *ptr != 12345 {
+		t.Errorf("bad %s uint: %#v ≠ %d\n", "pointer", ptr, 12345)
+	}
+
+	null := NewUint(0, false)
+	ptr = null.Ptr()
+	if ptr != nil {
+		t.Errorf("bad %s uint: %#v ≠ %s\n", "nil pointer", ptr, "nil")
+	}
+}
+
+func TestUintIsNull(t *testing.T) {
+	i := UintFrom(12345)
+	if i.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	null := NewUint(0, false)
+	if !null.IsNull() {
+		t.Errorf("IsNull() should be true")
+	}
+
+	zero := NewUint(0, true)
+	if zero.IsNull() {
+		t.Errorf("IsNull() should be false")
+	}
+
+	var testInt interface{}
+	testInt = zero
+	if _, ok := testInt.(Nullable); !ok {
+		t.Errorf("Nullable interface should be implemented")
+	}
+}
+
+func TestUintSetValid(t *testing.T) {
+	change := NewUint(0, false)
+	assertNullUint(t, change, "SetValid()")
+	change.SetValid(12345)
+	assertUint(t, change, "SetValid()")
+}
+
+func TestUintScan(t *testing.T) {
+	var i Uint
+	err := i.Scan(12345)
+	maybePanic(err)
+	assertUint(t, i, "scanned uint")
+
+	var null Uint
+	err = null.Scan(nil)
+	maybePanic(err)
+	assertNullUint(t, null, "scanned null")
+}
+
+func assertUint(t *testing.T, i Uint, from string) {
+	if i.Uint != 12345 {
+		t.Errorf("bad %s uint: %d ≠ %d\n", from, i.Uint, 12345)
+	}
+	if !i.Valid {
+		t.Error(from, "is invalid, but should be valid")
+	}
+}
+
+func assertNullUint(t *testing.T, i Uint, from string) {
+	if i.Valid {
+		t.Error(from, "is valid, but should be invalid")
+	}
+}

extras/orderedmap/ordered_map.go

@@ -0,0 +1,301 @@
+package orderedmap
+
+// mostly from https://github.com/iancoleman/orderedmap
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+)
+
+type keyIndex struct {
+	Key   string
+	Index int
+}
+
+type byIndex []keyIndex
+
+func (a byIndex) Len() int           { return len(a) }
+func (a byIndex) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byIndex) Less(i, j int) bool { return a[i].Index < a[j].Index }
+
+type Map struct {
+	l      sync.RWMutex
+	keys   []string
+	values map[string]interface{}
+}
+
+func New() *Map {
+	o := Map{}
+	o.l = sync.RWMutex{}
+	o.keys = []string{}
+	o.values = map[string]interface{}{}
+	return &o
+}
+
+func (o *Map) Get(key string) (interface{}, bool) {
+	o.l.RLock()
+	defer o.l.RUnlock()
+	val, exists := o.values[key]
+	return val, exists
+}
+
+func (o *Map) Set(key string, value interface{}) {
+	o.l.Lock()
+	defer o.l.Unlock()
+	_, exists := o.values[key]
+	if !exists {
+		o.keys = append(o.keys, key)
+	}
+	o.values[key] = value
+}
+
+const outOfRange = "position value %d is outside of the range %d - %d"
+
+//InsertAt This is a zero based position index: 0,1,2,3..n(from left) OR -1,-2,-3...-n(from right) where -1 is the last place.
+func (o *Map) InsertAt(key string, value interface{}, position int) error {
+	o.l.Lock()
+	defer o.l.Unlock()
+	var index = position
+	if position < 0 {
+		// support indexing from the back: -1 = last in array.
+		index += len(o.keys) + 1
+		if index < 0 || index > len(o.keys) {
+			return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1))
+		}
+	} else if index > len(o.keys) {
+		return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1))
+	}
+	_, exists := o.values[key]
+	if !exists {
+		// left + key + right
+		o.keys = append(o.keys[0:index], append([]string{key}, o.keys[index:]...)...)
+	}
+	o.values[key] = value
+
+	return nil
+}
+
+func (o *Map) Prepend(key string, value interface{}) {
+	o.l.Lock()
+	defer o.l.Unlock()
+	_, exists := o.values[key]
+	if !exists {
+		o.keys = append([]string{key}, o.keys...)
+	}
+	o.values[key] = value
+}
+
+func (o *Map) Delete(key string) {
+	o.l.Lock()
+	defer o.l.Unlock()
+	// check key is in use
+	_, ok := o.values[key]
+	if !ok {
+		return
+	}
+	// remove from keys
+	for i, k := range o.keys {
+		if k == key {
+			o.keys = append(o.keys[:i], o.keys[i+1:]...)
+			break
+		}
+	}
+	// remove from values
+	delete(o.values, key)
+}
+
+func (o *Map) Keys() []string {
+	o.l.RLock()
+	defer o.l.RUnlock()
+	return o.keys
+}
+
+func (o *Map) UnmarshalJSON(b []byte) error {
+	o.l.Lock()
+	defer o.l.Unlock()
+	m := map[string]interface{}{}
+	if err := json.Unmarshal(b, &m); err != nil {
+		return errors.Err(err)
+	}
+	s := string(b)
+	mapToOrderedMap(o, s, m)
+	return nil
+}
+
+func mapToOrderedMap(o *Map, s string, m map[string]interface{}) {
+	// Get the order of the keys
+	orderedKeys := []keyIndex{}
+	for k := range m {
+		kEscaped := strings.Replace(k, `"`, `\"`, -1)
+		kQuoted := `"` + kEscaped + `"`
+		// Find how much content exists before this key.
+		// If all content from this key and after is replaced with a close
+		// brace, it should still form a valid json string.
+		sTrimmed := s
+		for len(sTrimmed) > 0 {
+			lastIndex := strings.LastIndex(sTrimmed, kQuoted)
+			if lastIndex == -1 {
+				break
+			}
+			sTrimmed = sTrimmed[0:lastIndex]
+			sTrimmed = strings.TrimSpace(sTrimmed)
+			if len(sTrimmed) > 0 && sTrimmed[len(sTrimmed)-1] == ',' {
+				sTrimmed = sTrimmed[0 : len(sTrimmed)-1]
+			}
+			maybeValidJson := sTrimmed + "}"
+			testMap := map[string]interface{}{}
+			err := json.Unmarshal([]byte(maybeValidJson), &testMap)
+			if err == nil {
+				// record the position of this key in s
+				ki := keyIndex{
+					Key:   k,
+					Index: len(sTrimmed),
+				}
+				orderedKeys = append(orderedKeys, ki)
+				// shorten the string to get the next key
+				startOfValueIndex := lastIndex + len(kQuoted)
+				valueStr := s[startOfValueIndex : len(s)-1]
+				valueStr = strings.TrimSpace(valueStr)
+				if len(valueStr) > 0 && valueStr[0] == ':' {
+					valueStr = valueStr[1:]
+				}
+				valueStr = strings.TrimSpace(valueStr)
+				if valueStr[0] == '{' {
+					// if the value for this key is a map, convert it to an orderedmap.
+					// find end of valueStr by removing everything after last }
+					// until it forms valid json
+					hasValidJson := false
+					i := 1
+					for i < len(valueStr) && !hasValidJson {
+						if valueStr[i] != '}' {
+							i = i + 1
+							continue
+						}
+						subTestMap := map[string]interface{}{}
+						testValue := valueStr[0 : i+1]
+						err = json.Unmarshal([]byte(testValue), &subTestMap)
+						if err == nil {
+							hasValidJson = true
+							valueStr = testValue
+							break
+						}
+						i = i + 1
+					}
+					// convert to orderedmap
+					if hasValidJson {
+						mkTyped := m[k].(map[string]interface{})
+						oo := &Map{}
+						mapToOrderedMap(oo, valueStr, mkTyped)
+						m[k] = oo
+					}
+				} else if valueStr[0] == '[' {
+					// if the value for this key is a []interface, convert any map items to an orderedmap.
+					// find end of valueStr by removing everything after last ]
+					// until it forms valid json
+					hasValidJson := false
+					i := 1
+					for i < len(valueStr) && !hasValidJson {
+						if valueStr[i] != ']' {
+							i = i + 1
+							continue
+						}
+						subTestSlice := []interface{}{}
+						testValue := valueStr[0 : i+1]
+						err = json.Unmarshal([]byte(testValue), &subTestSlice)
+						if err == nil {
+							hasValidJson = true
+							valueStr = testValue
+							break
+						}
+						i = i + 1
+					}
+					if hasValidJson {
+						itemsStr := valueStr[1 : len(valueStr)-1]
+						// get next item in the slice
+						itemIndex := 0
+						startItem := 0
+						endItem := 0
+						for endItem < len(itemsStr) {
+							if itemsStr[endItem] != ',' && endItem < len(itemsStr)-1 {
+								endItem = endItem + 1
+								continue
+							}
+							// if this substring compiles to json, it's the next item
+							possibleItemStr := strings.TrimSpace(itemsStr[startItem:endItem])
+							var possibleItem interface{}
+							err = json.Unmarshal([]byte(possibleItemStr), &possibleItem)
+							if err != nil {
+								endItem = endItem + 1
+								continue
+							}
+							// if item is map, convert to orderedmap
+							if possibleItemStr[0] == '{' {
+								mkTyped := m[k].([]interface{})
+								mkiTyped := mkTyped[itemIndex].(map[string]interface{})
+								oo := &Map{}
+								mapToOrderedMap(oo, possibleItemStr, mkiTyped)
+								// replace original map with orderedmap
+								mkTyped[itemIndex] = oo
+								m[k] = mkTyped
+							}
+							// remove this item from itemsStr
+							startItem = endItem + 1
+							endItem = endItem + 1
+							itemIndex = itemIndex + 1
+						}
+					}
+				}
+				break
+			}
+		}
+	}
+	// Sort the keys
+	sort.Sort(byIndex(orderedKeys))
+	// Convert sorted keys to string slice
+	k := []string{}
+	for _, ki := range orderedKeys {
+		k = append(k, ki.Key)
+	}
+	// Set the Map values
+	o.values = m
+	o.keys = k
+}
+
+func (o *Map) Copy() *Map {
+	new := New()
+
+	for _, k := range o.keys {
+		v, _ := o.Get(k)
+		new.Set(k, v)
+	}
+
+	return new
+}
+
+func (o *Map) MarshalJSON() ([]byte, error) {
+	o.l.RLock()
+	defer o.l.RUnlock()
+	s := "{"
+	for _, k := range o.keys {
+		// add key
+		kEscaped := strings.Replace(k, `"`, `\"`, -1)
+		s = s + `"` + kEscaped + `":`
+		// add value
+		v := o.values[k]
+		vBytes, err := json.Marshal(v)
+		if err != nil {
+			return []byte{}, errors.Err(err)
+		}
+		s = s + string(vBytes) + ","
+	}
+	if len(o.keys) > 0 {
+		s = s[0 : len(s)-1]
+	}
+	s = s + "}"
+	return []byte(s), nil
+}

extras/orderedmap/ordered_map_test.go

@@ -0,0 +1,478 @@
+package orderedmap
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/spf13/cast"
+)
+
+func TestOrderedMap(t *testing.T) {
+	o := New()
+	// number
+	o.Set("number", 3)
+	v, _ := o.Get("number")
+	if v.(int) != 3 {
+		t.Error("Set number")
+	}
+	// string
+	o.Set("string", "x")
+	v, _ = o.Get("string")
+	if v.(string) != "x" {
+		t.Error("Set string")
+	}
+	// string slice
+	o.Set("strings", []string{
+		"t",
+		"u",
+	})
+	v, _ = o.Get("strings")
+	if v.([]string)[0] != "t" {
+		t.Error("Set strings first index")
+	}
+	if v.([]string)[1] != "u" {
+		t.Error("Set strings second index")
+	}
+	// mixed slice
+	o.Set("mixed", []interface{}{
+		1,
+		"1",
+	})
+	v, _ = o.Get("mixed")
+	if v.([]interface{})[0].(int) != 1 {
+		t.Error("Set mixed int")
+	}
+	if v.([]interface{})[1].(string) != "1" {
+		t.Error("Set mixed string")
+	}
+	// overriding existing key
+	o.Set("number", 4)
+	v, _ = o.Get("number")
+	if v.(int) != 4 {
+		t.Error("Override existing key")
+	}
+	// Keys method
+	keys := o.Keys()
+	expectedKeys := []string{
+		"number",
+		"string",
+		"strings",
+		"mixed",
+	}
+	for i := range keys {
+		if keys[i] != expectedKeys[i] {
+			t.Error("Keys method", keys[i], "!=", expectedKeys[i])
+		}
+	}
+	for i := range expectedKeys {
+		if keys[i] != expectedKeys[i] {
+			t.Error("Keys method", keys[i], "!=", expectedKeys[i])
+		}
+	}
+	// delete
+	o.Delete("strings")
+	o.Delete("not a key being used")
+	if len(o.Keys()) != 3 {
+		t.Error("Delete method")
+	}
+	_, ok := o.Get("strings")
+	if ok {
+		t.Error("Delete did not remove 'strings' key")
+	}
+}
+
+func TestBlankMarshalJSON(t *testing.T) {
+	o := New()
+	// blank map
+	b, err := json.Marshal(o)
+	if err != nil {
+		t.Error("Marshalling blank map to json", err)
+	}
+	s := string(b)
+	// check json is correctly ordered
+	if s != `{}` {
+		t.Error("JSON Marshaling blank map value is incorrect", s)
+	}
+	// convert to indented json
+	bi, err := json.MarshalIndent(o, "", "  ")
+	if err != nil {
+		t.Error("Marshalling indented json for blank map", err)
+	}
+	si := string(bi)
+	ei := `{}`
+	if si != ei {
+		fmt.Println(ei)
+		fmt.Println(si)
+		t.Error("JSON MarshalIndent blank map value is incorrect", si)
+	}
+}
+
+func TestMarshalJSON(t *testing.T) {
+	o := New()
+	// number
+	o.Set("number", 3)
+	// string
+	o.Set("string", "x")
+	// new value keeps key in old position
+	o.Set("number", 4)
+	// keys not sorted alphabetically
+	o.Set("z", 1)
+	o.Set("a", 2)
+	o.Set("b", 3)
+	// slice
+	o.Set("slice", []interface{}{
+		"1",
+		1,
+	})
+	// orderedmap
+	v := New()
+	v.Set("e", 1)
+	v.Set("a", 2)
+	o.Set("orderedmap", v)
+	// double quote in key
+	o.Set(`test"ing`, 9)
+	// convert to json
+	b, err := json.Marshal(o)
+	if err != nil {
+		t.Error("Marshalling json", err)
+	}
+	s := string(b)
+	// check json is correctly ordered
+	if s != `{"number":4,"string":"x","z":1,"a":2,"b":3,"slice":["1",1],"orderedmap":{"e":1,"a":2},"test\"ing":9}` {
+		t.Error("JSON Marshal value is incorrect", s)
+	}
+	// convert to indented json
+	bi, err := json.MarshalIndent(o, "", "  ")
+	if err != nil {
+		t.Error("Marshalling indented json", err)
+	}
+	si := string(bi)
+	ei := `{
+  "number": 4,
+  "string": "x",
+  "z": 1,
+  "a": 2,
+  "b": 3,
+  "slice": [
+    "1",
+    1
+  ],
+  "orderedmap": {
+    "e": 1,
+    "a": 2
+  },
+  "test\"ing": 9
+}`
+	if si != ei {
+		fmt.Println(ei)
+		fmt.Println(si)
+		t.Error("JSON MarshalIndent value is incorrect", si)
+	}
+}
+
+func TestUnmarshalJSON(t *testing.T) {
+	s := `{
+  "number": 4,
+  "string": "x",
+  "z": 1,
+  "a": "should not break with unclosed { character in value",
+  "b": 3,
+  "slice": [
+    "1",
+    1
+  ],
+  "orderedmap": {
+    "e": 1,
+    "a { nested key with brace": "with a }}}} }} {{{ brace value",
+	"after": {
+		"link": "test {{{ with even deeper nested braces }"
+	}
+  },
+  "test\"ing": 9,
+  "after": 1,
+  "multitype_array": [
+    "test",
+	1,
+	{ "map": "obj", "it" : 5, ":colon in key": "colon: in value" }
+  ],
+  "should not break with { character in key": 1
+}`
+	o := New()
+	err := json.Unmarshal([]byte(s), &o)
+	if err != nil {
+		t.Error("JSON Unmarshal error", err)
+	}
+	// Check the root keys
+	expectedKeys := []string{
+		"number",
+		"string",
+		"z",
+		"a",
+		"b",
+		"slice",
+		"orderedmap",
+		"test\"ing",
+		"after",
+		"multitype_array",
+		"should not break with { character in key",
+	}
+	k := o.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// Check nested maps are converted to orderedmaps
+	// nested 1 level deep
+	expectedKeys = []string{
+		"e",
+		"a { nested key with brace",
+		"after",
+	}
+	vi, ok := o.Get("orderedmap")
+	if !ok {
+		t.Error("Missing key for nested map 1 deep")
+	}
+	v := vi.(*Map)
+	k = v.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// nested 2 levels deep
+	expectedKeys = []string{
+		"link",
+	}
+	vi, ok = v.Get("after")
+	if !ok {
+		t.Error("Missing key for nested map 2 deep")
+	}
+	v = vi.(*Map)
+	k = v.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// multitype array
+	expectedKeys = []string{
+		"map",
+		"it",
+		":colon in key",
+	}
+	vislice, ok := o.Get("multitype_array")
+	if !ok {
+		t.Error("Missing key for multitype array")
+	}
+	vslice := vislice.([]interface{})
+	vmap := vslice[2].(*Map)
+	k = vmap.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+}
+
+func TestUnmarshalJSONSpecialChars(t *testing.T) {
+	s := `{ " \\\\\\\\\\\\ "  : { "\\\\\\" : "\\\\\"\\" }, "\\":  " \\\\ test " }`
+	o := New()
+	err := json.Unmarshal([]byte(s), &o)
+	if err != nil {
+		t.Error("JSON Unmarshal error with special chars", err)
+	}
+}
+
+func TestUnmarshalJSONArrayOfMaps(t *testing.T) {
+	s := `
+{
+  "name": "test",
+  "percent": 6,
+  "breakdown": [
+    {
+      "name": "a",
+      "percent": 0.9
+    },
+    {
+      "name": "b",
+      "percent": 0.9
+    },
+    {
+      "name": "d",
+      "percent": 0.4
+    },
+    {
+      "name": "e",
+      "percent": 2.7
+    }
+  ]
+}
+`
+	o := New()
+	err := json.Unmarshal([]byte(s), &o)
+	if err != nil {
+		t.Error("JSON Unmarshal error", err)
+	}
+	// Check the root keys
+	expectedKeys := []string{
+		"name",
+		"percent",
+		"breakdown",
+	}
+	k := o.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// Check nested maps are converted to orderedmaps
+	// nested 1 level deep
+	expectedKeys = []string{
+		"name",
+		"percent",
+	}
+	vi, ok := o.Get("breakdown")
+	if !ok {
+		t.Error("Missing key for nested map 1 deep")
+	}
+	vs := vi.([]interface{})
+	for _, vInterface := range vs {
+		v := vInterface.(*Map)
+		k = v.Keys()
+		for i := range k {
+			if k[i] != expectedKeys[i] {
+				t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i])
+			}
+		}
+	}
+}
+
+func TestInsertAt(t *testing.T) {
+	om := New()
+	om.Set("zero", 0)
+	om.Set("one", 1)
+	om.Set("two", 2)
+
+	err := om.InsertAt("TEST", 10000, 4) //3 is this added one in size of map
+	if err == nil {
+		t.Error("expected insert at greater position than size of map to produce error")
+	}
+
+	err = om.InsertAt("A", 100, 2)
+	if err != nil {
+		t.Error(err)
+	}
+	// Test it's at end
+	if om.values[om.keys[2]] != 100 {
+		t.Error("expected entry A to be at position 2", om.keys)
+	}
+	if om.values[om.keys[3]] != 2 {
+		t.Error("expected two to be in position 1", om.keys)
+	}
+
+	err = om.InsertAt("B", 200, 0)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if om.values[om.keys[0]] != 200 {
+		t.Error("expected B to be position 0", om.keys)
+	}
+
+	err = om.InsertAt("C", 300, -1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Should show up at the end
+	if om.values[om.keys[len(om.keys)-1]] != 300 {
+		t.Error(fmt.Sprintf("expected C to be in position %d", len(om.keys)-1), om.keys)
+	}
+
+	err = om.InsertAt("D", 400, 1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if om.values[om.keys[1]] != 400 {
+		t.Error("expceted D to be position 1", om.keys)
+	}
+
+	err = om.InsertAt("F", 600, -8)
+	if err != nil {
+		t.Error(err)
+	}
+	if om.values[om.keys[0]] != 600 {
+		t.Error("expected F to be in position 0", om.keys)
+	}
+}
+
+func TestConcurrency(t *testing.T) {
+	wg := sync.WaitGroup{}
+	type concurrency struct {
+		a string
+		b int
+		c time.Time
+		d bool
+	}
+
+	//Starting Map
+	m := New()
+	m.Set("A", concurrency{"string", 10, time.Now(), true})
+	m.Set("B", concurrency{"string", 10, time.Now(), true})
+	m.Set("C", concurrency{"string", 10, time.Now(), true})
+	m.Set("D", concurrency{"string", 10, time.Now(), true})
+	m.Set("E", concurrency{"string", 10, time.Now(), true})
+	m.Set("F", concurrency{"string", 10, time.Now(), true})
+	m.Set("G", concurrency{"string", 10, time.Now(), true})
+	m.Set("H", concurrency{"string", 10, time.Now(), true})
+	//Inserts
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 50; i++ {
+			wg.Add(1)
+			go func(index int) {
+				defer wg.Done()
+				m.Set("New"+strconv.Itoa(index), concurrency{"string", index, time.Now(), cast.ToBool(index % 2)})
+			}(i)
+		}
+	}()
+	//Reads
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 50; i++ {
+			wg.Add(1)
+			go func(index int) {
+				defer wg.Done()
+				_, _ = m.Get("New" + strconv.Itoa(rand.Intn(99)))
+			}(i)
+		}
+	}()
+	//Marshalling like endpoint
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		for i := 0; i < 50; i++ {
+			wg.Add(1)
+			go func(index int) {
+				defer wg.Done()
+				_, err := m.MarshalJSON()
+				if err != nil {
+					t.Error(err)
+				}
+			}(i)
+		}
+	}()
+
+	wg.Wait()
+
+}

extras/query/query.go

@@ -0,0 +1,123 @@
+package query
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/null"
+)
+
+func InterpolateParams(query string, args ...interface{}) (string, error) {
+	for i := 0; i < len(args); i++ {
+		field := reflect.ValueOf(args[i])
+
+		if value, ok := field.Interface().(time.Time); ok {
+			query = strings.Replace(query, "?", `"`+value.Format("2006-01-02 15:04:05")+`"`, 1)
+		} else if nullable, ok := field.Interface().(null.Nullable); ok {
+			if nullable.IsNull() {
+				query = strings.Replace(query, "?", "NULL", 1)
+			} else {
+				switch field.Type() {
+				case reflect.TypeOf(null.Time{}):
+					query = strings.Replace(query, "?", `"`+field.Interface().(null.Time).Time.Format("2006-01-02 15:04:05")+`"`, 1)
+				case reflect.TypeOf(null.Int{}):
+					query = strings.Replace(query, "?", strconv.FormatInt(int64(field.Interface().(null.Int).Int), 10), 1)
+				case reflect.TypeOf(null.Int8{}):
+					query = strings.Replace(query, "?", strconv.FormatInt(int64(field.Interface().(null.Int8).Int8), 10), 1)
+				case reflect.TypeOf(null.Int16{}):
+					query = strings.Replace(query, "?", strconv.FormatInt(int64(field.Interface().(null.Int16).Int16), 10), 1)
+				case reflect.TypeOf(null.Int32{}):
+					query = strings.Replace(query, "?", strconv.FormatInt(int64(field.Interface().(null.Int32).Int32), 10), 1)
+				case reflect.TypeOf(null.Int64{}):
+					query = strings.Replace(query, "?", strconv.FormatInt(field.Interface().(null.Int64).Int64, 10), 1)
+				case reflect.TypeOf(null.Uint{}):
+					query = strings.Replace(query, "?", strconv.FormatUint(uint64(field.Interface().(null.Uint).Uint), 10), 1)
+				case reflect.TypeOf(null.Uint8{}):
+					query = strings.Replace(query, "?", strconv.FormatUint(uint64(field.Interface().(null.Uint8).Uint8), 10), 1)
+				case reflect.TypeOf(null.Uint16{}):
+					query = strings.Replace(query, "?", strconv.FormatUint(uint64(field.Interface().(null.Uint16).Uint16), 10), 1)
+				case reflect.TypeOf(null.Uint32{}):
+					query = strings.Replace(query, "?", strconv.FormatUint(uint64(field.Interface().(null.Uint32).Uint32), 10), 1)
+				case reflect.TypeOf(null.Uint64{}):
+					query = strings.Replace(query, "?", strconv.FormatUint(field.Interface().(null.Uint64).Uint64, 10), 1)
+				case reflect.TypeOf(null.String{}):
+					query = strings.Replace(query, "?", `"`+field.Interface().(null.String).String+`"`, 1)
+				case reflect.TypeOf(null.Bool{}):
+					if field.Interface().(null.Bool).Bool {
+						query = strings.Replace(query, "?", "1", 1)
+					} else {
+						query = strings.Replace(query, "?", "0", 1)
+					}
+				}
+			}
+		} else {
+			switch field.Kind() {
+			case reflect.Bool:
+				boolString := "0"
+				if field.Bool() {
+					boolString = "1"
+				}
+				query = strings.Replace(query, "?", boolString, 1)
+			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+				query = strings.Replace(query, "?", strconv.FormatInt(field.Int(), 10), 1)
+			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+				query = strings.Replace(query, "?", strconv.FormatUint(field.Uint(), 10), 1)
+			case reflect.Float32, reflect.Float64:
+				query = strings.Replace(query, "?", strconv.FormatFloat(field.Float(), 'f', -1, 64), 1)
+			case reflect.String:
+				query = strings.Replace(query, "?", `"`+field.String()+`"`, 1)
+			case reflect.Slice:
+				query = strings.Replace(query, "?", `x`+fmt.Sprintf("%0x", field), 1)
+			default:
+				return "", errors.Err("dont know how to interpolate type " + field.Type().String())
+			}
+		}
+	}
+
+	// tabs to spaces, for easier copying into mysql prompt
+	query = strings.Replace(query, "\t", "    ", -1)
+
+	return query, nil
+}
+
+// Qs is a shortcut for one group of positional placeholders
+func Qs(count int) string {
+	return Placeholders(false, count, 1, 1)
+}
+
+// placeholders creates indexed or positional placeholders, in groups, with different starts
+func Placeholders(indexPlaceholders bool, count int, start int, group int) string {
+	buf := bytes.Buffer{}
+
+	if start == 0 || group == 0 {
+		panic("invalid start or group numbers supplied.")
+	}
+
+	if group > 1 {
+		buf.WriteByte('(')
+	}
+	for i := 0; i < count; i++ {
+		if i != 0 {
+			if group > 1 && i%group == 0 {
+				buf.WriteString("),(")
+			} else {
+				buf.WriteByte(',')
+			}
+		}
+		if indexPlaceholders {
+			buf.WriteString(fmt.Sprintf("$%d", start+i))
+		} else {
+			buf.WriteByte('?')
+		}
+	}
+	if group > 1 {
+		buf.WriteByte(')')
+	}
+
+	return buf.String()
+}

extras/stop/stop.go

@@ -23,7 +23,7 @@ import (
 // Chan is a receive-only channel
 type Chan <-chan struct{}
 
-// Group extends sync.WaitGroup to add a convenient way to stop running goroutines
+// Stopper extends sync.WaitGroup to add a convenient way to stop running goroutines
 type Group struct {
 	sync.WaitGroup
 	ctx    context.Context

extras/travis/travis.go

@@ -0,0 +1,112 @@
+package travis
+
+/*
+Copyright 2017 Shapath Neupane (@theshapguy)
+Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
+1. Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
+2. Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
+3. Neither the name of the copyright holder nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission.
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+------------------------------------------------
+Listener - written in Go because it's native web server is much more robust than Python. Plus its fun to write Go!
+NOTE: Make sure you are using the right domain for travis [.com] or [.org]
+Modified by wilsonk@lbry.io for LBRY internal-apis
+*/
+
+import (
+	"crypto"
+	"crypto/rsa"
+	"crypto/sha1"
+	"crypto/x509"
+	"encoding/base64"
+	"encoding/json"
+	"encoding/pem"
+	"net/http"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+)
+
+func publicKey(isPrivateRepo bool) (*rsa.PublicKey, error) {
+	var response *http.Response
+	var err error
+	if !isPrivateRepo {
+		response, err = http.Get("https://api.travis-ci.org/config")
+	} else {
+		response, err = http.Get("https://api.travis-ci.com/config")
+	}
+	if err != nil {
+		return nil, errors.Err("cannot fetch travis public key")
+	}
+	defer response.Body.Close()
+
+	type configKey struct {
+		Config struct {
+			Notifications struct {
+				Webhook struct {
+					PublicKey string `json:"public_key"`
+				} `json:"webhook"`
+			} `json:"notifications"`
+		} `json:"config"`
+	}
+
+	var t configKey
+
+	decoder := json.NewDecoder(response.Body)
+	err = decoder.Decode(&t)
+	if err != nil {
+		return nil, errors.Err("cannot decode travis public key")
+	}
+
+	keyBlock, _ := pem.Decode([]byte(t.Config.Notifications.Webhook.PublicKey))
+	if keyBlock == nil || keyBlock.Type != "PUBLIC KEY" {
+		return nil, errors.Err("invalid travis public key")
+	}
+
+	publicKey, err := x509.ParsePKIXPublicKey(keyBlock.Bytes)
+	if err != nil {
+		return nil, errors.Err("invalid travis public key")
+	}
+
+	return publicKey.(*rsa.PublicKey), nil
+}
+
+func payloadDigest(payload string) []byte {
+	hash := sha1.New()
+	hash.Write([]byte(payload))
+	return hash.Sum(nil)
+}
+
+func ValidateSignature(isPrivateRepo bool, r *http.Request) error {
+	key, err := publicKey(isPrivateRepo)
+	if err != nil {
+		return errors.Err(err)
+	}
+
+	signature, err := base64.StdEncoding.DecodeString(r.Header.Get("Signature"))
+	if err != nil {
+		return errors.Err("cannot decode signature")
+	}
+
+	payload := payloadDigest(r.FormValue("payload"))
+
+	err = rsa.VerifyPKCS1v15(key, crypto.SHA1, payload, signature)
+	if err != nil {
+		if err == rsa.ErrVerification {
+			return errors.Err("invalid payload signature")
+		}
+		return errors.Err(err)
+	}
+
+	return nil
+}
+
+func NewFromRequest(r *http.Request) (*Webhook, error) {
+	w := new(Webhook)
+
+	err := json.Unmarshal([]byte(r.FormValue("payload")), w)
+	if err != nil {
+		return nil, errors.Err(err)
+	}
+
+	return w, nil
+}

extras/travis/webhook.go

@@ -0,0 +1,66 @@
+package travis
+
+import "time"
+
+// https://docs.travis-ci.com/user/notifications/#Webhooks-Delivery-Format
+
+const (
+	statusSuccess    = 0
+	statusNotSuccess = 1
+)
+
+type Webhook struct {
+	ID                int        `json:"id"`
+	Number            string     `json:"number"`
+	Type              string     `json:"type"`
+	State             string     `json:"state"`
+	Status            int        `json:"status"` // status and result are the same
+	Result            int        `json:"result"`
+	StatusMessage     string     `json:"status_message"` // status_message and result_message are the same
+	ResultMessage     string     `json:"result_message"`
+	StartedAt         time.Time  `json:"started_at"`
+	FinishedAt        time.Time  `json:"finished_at"`
+	Duration          int        `json:"duration"`
+	BuildURL          string     `json:"build_url"`
+	CommitID          int        `json:"commit_id"`
+	Commit            string     `json:"commit"`
+	BaseCommit        string     `json:"base_commit"`
+	HeadCommit        string     `json:"head_commit"`
+	Branch            string     `json:"branch"`
+	Message           string     `json:"message"`
+	CompareURL        string     `json:"compare_url"`
+	CommittedAt       time.Time  `json:"committed_at"`
+	AuthorName        string     `json:"author_name"`
+	AuthorEmail       string     `json:"author_email"`
+	CommitterName     string     `json:"committer_name"`
+	CommitterEmail    string     `json:"committer_email"`
+	PullRequest       bool       `json:"pull_request"`
+	PullRequestNumber int        `json:"pull_request_number"`
+	PullRequestTitle  string     `json:"pull_request_title"`
+	Tag               string     `json:"tag"`
+	Repository        Repository `json:"repository"`
+}
+
+type Repository struct {
+	ID        int    `json:"id"`
+	Name      string `json:"name"`
+	OwnerName string `json:"owner_name"`
+	URL       string `json:"url"`
+}
+
+// IsMatch make sure the webhook is for you...
+func (w Webhook) IsMatch(branch string, repo string, owner string) bool {
+	return w.Branch == branch &&
+		w.Repository.Name == repo &&
+		w.Repository.OwnerName == owner
+}
+
+func (w Webhook) ShouldDeploy() bool {
+	// when travis builds a pull request, Branch is the target branch, not the origin branch
+	// source: https://docs.travis-ci.com/user/environment-variables/#Default-Environment-Variables
+	return w.Status == statusSuccess && w.Branch == "master" && !w.PullRequest
+}
+
+func (w Webhook) DeploySummary() string {
+	return w.Commit[:8] + ": " + w.Message
+}

extras/util/pointer.go

@@ -0,0 +1,59 @@
+// mostly copied from github.com/AlekSi/pointer
+// Provides helpers to get pointers to values of build-in types.
+
+package util
+
+import (
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/null"
+)
+
+func PtrToBool(b bool) *bool                   { return &b }
+func PtrToByte(b byte) *byte                   { return &b }
+func PtrToComplex128(c complex128) *complex128 { return &c }
+func PtrToComplex64(c complex64) *complex64    { return &c }
+func PtrToError(e error) *error                { return &e }
+func PtrToFloat32(f float32) *float32          { return &f }
+func PtrToFloat64(f float64) *float64          { return &f }
+func PtrToInt(i int) *int                      { return &i }
+func PtrToInt16(i int16) *int16                { return &i }
+func PtrToInt32(i int32) *int32                { return &i }
+func PtrToInt64(i int64) *int64                { return &i }
+func PtrToInt8(i int8) *int8                   { return &i }
+func PtrToRune(r rune) *rune                   { return &r }
+func PtrToString(s string) *string             { return &s }
+func PtrToTime(t time.Time) *time.Time         { return &t }
+func PtrToUint(u uint) *uint                   { return &u }
+func PtrToUint16(u uint16) *uint16             { return &u }
+func PtrToUint32(u uint32) *uint32             { return &u }
+func PtrToUint64(u uint64) *uint64             { return &u }
+func PtrToUint8(u uint8) *uint8                { return &u }
+func PtrToUintptr(u uintptr) *uintptr          { return &u }
+
+func PtrToNullString(s string) *null.String    { n := null.StringFrom(s); return &n }
+func PtrToNullUint64(u uint64) *null.Uint64    { n := null.Uint64From(u); return &n }
+func PtrToNullTime(t time.Time) *null.Time     { n := null.TimeFrom(t); return &n }
+func PtrToNullFloat64(f float64) *null.Float64 { n := null.Float64From(f); return &n }
+func PtrToNullFloat32(f float32) *null.Float32 { n := null.Float32From(f); return &n }
+
+func StrFromPtr(ptr *string) string {
+	if ptr == nil {
+		return ""
+	}
+	return *ptr
+}
+
+func StrFromNull(str null.String) string {
+	if !str.Valid {
+		return ""
+	}
+	return str.String
+}
+
+func NullStringFrom(s string) null.String {
+	if s == "" {
+		return null.String{}
+	}
+	return null.StringFrom(s)
+}

extras/util/slack.go

@@ -0,0 +1,97 @@
+package util
+
+import (
+	"fmt"
+	"net"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/slack-go/slack"
+)
+
+var defaultChannel string
+var defaultUsername string
+var slackApi *slack.Client
+
+// InitSlack Initializes a slack client with the given token and sets the default channel.
+func InitSlack(token string, channel string, username string) {
+	c := &http.Client{
+		Transport: &http.Transport{
+			DialContext: (&net.Dialer{
+				Timeout:   30 * time.Second,
+				KeepAlive: 30 * time.Second,
+			}).DialContext,
+			TLSHandshakeTimeout:   10 * time.Second,
+			ResponseHeaderTimeout: 10 * time.Second,
+			ExpectContinueTimeout: 1 * time.Second,
+		},
+	}
+	slackApi = slack.New(token, slack.OptionHTTPClient(c))
+	defaultChannel = channel
+	defaultUsername = username
+}
+
+// SendToSlackUser Sends message to a specific user.
+func SendToSlackUser(user, username, format string, a ...interface{}) error {
+	message := format
+	if len(a) > 0 {
+		message = fmt.Sprintf(format, a...)
+	}
+	if !strings.HasPrefix(user, "@") {
+		user = "@" + user
+	}
+	return sendToSlack(user, username, message)
+}
+
+// SendToSlackChannel Sends message to a specific channel.
+func SendToSlackChannel(channel, username, format string, a ...interface{}) error {
+	message := format
+	if len(a) > 0 {
+		message = fmt.Sprintf(format, a...)
+	}
+	if !strings.HasPrefix(channel, "#") {
+		channel = "#" + channel
+	}
+	return sendToSlack(channel, username, message)
+}
+
+// SendToSlack Sends message to the default channel.
+func SendToSlack(format string, a ...interface{}) error {
+	message := format
+	if len(a) > 0 {
+		message = fmt.Sprintf(format, a...)
+	}
+	if defaultChannel == "" {
+		return errors.Err("no default slack channel set")
+	}
+
+	return sendToSlack(defaultChannel, defaultUsername, message)
+}
+
+func sendToSlack(channel, username, message string) error {
+	var err error
+
+	if slackApi == nil {
+		err = errors.Err("no slack token provided")
+	} else {
+		log.Debugln("slack: " + channel + ": " + message)
+		for {
+			_, _, err = slackApi.PostMessage(channel, slack.MsgOptionText(message, false), slack.MsgOptionUsername(username))
+			if err != nil && strings.Contains(err.Error(), "timeout awaiting response headers") {
+				continue
+			}
+			break
+		}
+	}
+
+	if err != nil {
+		log.Errorln("error sending to slack: " + err.Error())
+		return err
+	}
+
+	return nil
+}

extras/util/slice.go

@@ -0,0 +1,22 @@
+package util
+
+import "strings"
+
+func InSlice(str string, values []string) bool {
+	for _, v := range values {
+		if str == v {
+			return true
+		}
+	}
+	return false
+}
+
+// SubstringInSlice returns true if str is contained within any element of the values slice. False otherwise
+func SubstringInSlice(str string, values []string) bool {
+	for _, v := range values {
+		if strings.Contains(str, v) {
+			return true
+		}
+	}
+	return false
+}

extras/util/strings.go

@@ -0,0 +1,53 @@
+package util
+
+import (
+	"encoding/hex"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/unicode/norm"
+	"strings"
+)
+
+func StringSplitArg(stringToSplit, separator string) []interface{} {
+	split := strings.Split(stringToSplit, separator)
+	splitInterface := make([]interface{}, len(split))
+	for i, s := range split {
+		splitInterface[i] = s
+	}
+	return splitInterface
+}
+
+// NormalizeName Normalize names to remove weird characters and account to capitalization
+func NormalizeName(s string) string {
+	c := cases.Fold()
+	return c.String(norm.NFD.String(s))
+}
+
+// ReverseBytesInPlace reverse the bytes. thanks, Satoshi 😒
+func ReverseBytesInPlace(s []byte) {
+	for i, j := 0, len(s)-1; i < j; i, j = i+1, j-1 {
+		s[i], s[j] = s[j], s[i]
+	}
+}
+
+// TxIdToTxHash convert the txid to a hash for returning from the hub
+func TxIdToTxHash(txid string) []byte {
+	t, err := hex.DecodeString(txid)
+	if err != nil {
+		return nil
+	}
+
+	ReverseBytesInPlace(t)
+
+	return t
+}
+
+// TxHashToTxId convert the txHash from the response format back to an id
+func TxHashToTxId(txHash []byte) string {
+	t := make([]byte, len(txHash))
+	copy(t, txHash)
+
+	ReverseBytesInPlace(t)
+
+	return hex.EncodeToString(t)
+
+}

extras/util/underscore.go

@@ -0,0 +1,33 @@
+package util
+
+// https://github.com/go-pg/pg/blob/7c2d9d39a5cfc18a422c88a9f5f39d8d2cd10030/internal/underscore.go
+
+func isUpper(c byte) bool {
+	return c >= 'A' && c <= 'Z'
+}
+
+func isLower(c byte) bool {
+	return !isUpper(c)
+}
+
+func toLower(c byte) byte {
+	return c + 32
+}
+
+// Underscore converts "CamelCasedString" to "camel_cased_string".
+func Underscore(s string) string {
+	r := make([]byte, 0, len(s))
+	for i := 0; i < len(s); i++ {
+		c := s[i]
+		if isUpper(c) {
+			if i > 0 && i+1 < len(s) && (isLower(s[i-1]) || isLower(s[i+1])) {
+				r = append(r, '_', toLower(c))
+			} else {
+				r = append(r, toLower(c))
+			}
+		} else {
+			r = append(r, c)
+		}
+	}
+	return string(r)
+}

extras/validator/bool.go

@@ -0,0 +1,34 @@
+package validator
+
+var (
+	truthyValues = []string{"1", "yes", "y", "true"}
+	falseyValues = []string{"0", "no", "n", "false"}
+)
+
+// todo: consider using strconv.ParseBool instead
+
+func IsTruthy(value string) bool {
+	for _, e := range truthyValues {
+		if e == value {
+			return true
+		}
+	}
+	return false
+}
+
+func IsFalsey(value string) bool {
+	for _, e := range falseyValues {
+		if e == value {
+			return true
+		}
+	}
+	return false
+}
+
+func IsBoolString(value string) bool {
+	return IsTruthy(value) || IsFalsey(value)
+}
+
+func GetBoolStringValues() []string {
+	return append(truthyValues, falseyValues...)
+}

go.mod

@@ -1,28 +1,55 @@
-module github.com/lbryio/lbry.go/v3
+go 1.18
+
+module github.com/lbryio/lbry.go/v2
+
+replace github.com/btcsuite/btcd => github.com/lbryio/lbrycrd.go v0.0.0-20200203050410-e1076f12bf19
 
 require (
-	github.com/cockroachdb/errors v1.9.0
+	github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32
+	github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d
 	github.com/davecgh/go-spew v1.1.1
-	github.com/go-errors/errors v1.1.1 // indirect
+	github.com/fatih/structs v1.1.0
-	github.com/go-ini/ini v1.48.0
+	github.com/go-errors/errors v1.4.2
+	github.com/go-ini/ini v1.67.0
 	github.com/golang/protobuf v1.5.2
-	github.com/gorilla/mux v1.7.3
+	github.com/gorilla/mux v1.8.0
 	github.com/gorilla/rpc v1.2.0
-	github.com/lbryio/lbcd v0.22.117
+	github.com/lbryio/ozzo-validation v3.0.3-0.20170512160344-202201e212ec+incompatible
-	github.com/lbryio/lbcutil v1.0.202
+	github.com/lbryio/types v0.0.0-20220224142228-73610f6654a6
-	github.com/lbryio/lbry.go/v2 v2.7.1
-	github.com/lbryio/types v0.0.0-20201019032447-f0b4476ef386
 	github.com/lyoshenka/bencode v0.0.0-20180323155644-b7abd7672df5
-	github.com/sebdah/goldie v0.0.0-20190531093107-d313ffb52c77
+	github.com/mitchellh/mapstructure v1.5.0
-	github.com/sergi/go-diff v1.1.0
+	github.com/sebdah/goldie v1.0.0
-	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
+	github.com/sergi/go-diff v1.3.1
-	github.com/sirupsen/logrus v1.8.1
+	github.com/shopspring/decimal v1.3.1
-	github.com/spf13/cast v1.3.0
+	github.com/sirupsen/logrus v1.9.0
-	github.com/stretchr/testify v1.7.1
+	github.com/slack-go/slack v0.12.1
-	go.uber.org/atomic v1.7.0
+	github.com/spf13/cast v1.5.0
-	golang.org/x/crypto v0.0.0-20220518034528-6f7dac969898
+	github.com/stretchr/testify v1.8.2
-	golang.org/x/time v0.0.0-20201208040808-7e3f01d25324
+	github.com/ybbus/jsonrpc/v2 v2.1.7
+	go.uber.org/atomic v1.10.0
+	golang.org/x/crypto v0.7.0
+	golang.org/x/net v0.8.0
+	golang.org/x/oauth2 v0.6.0
+	golang.org/x/text v0.8.0
+	golang.org/x/time v0.3.0
+	google.golang.org/grpc v1.53.0
+	gopkg.in/nullbio/null.v6 v6.0.0-20161116030900-40264a2e6b79
 	gotest.tools v2.2.0+incompatible
 )
 
-go 1.16
+require (
+	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f // indirect
+	github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd // indirect
+	github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792 // indirect
+	github.com/google/go-cmp v0.5.9 // indirect
+	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/onsi/gomega v1.7.0 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	golang.org/x/sys v0.6.0 // indirect
+	google.golang.org/appengine v1.6.7 // indirect
+	google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+)

lbrycrd/address.go

@@ -3,13 +3,12 @@ package lbrycrd
 import (
 	"encoding/hex"
 
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+
+	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcutil"
+	"github.com/btcsuite/btcutil/base58"
 	"golang.org/x/crypto/ripemd160"
-
-	"github.com/lbryio/lbcd/chaincfg"
-	"github.com/lbryio/lbcutil"
-	"github.com/lbryio/lbcutil/base58"
-
-	"github.com/cockroachdb/errors"
 )
 
 // DecodeAddress decodes the string encoding of an address and returns
@@ -18,7 +17,7 @@ import (
 // The bitcoin network the address is associated with is extracted if possible.
 // When the address does not encode the network, such as in the case of a raw
 // public key, the address will be associated with the passed defaultNet.
-func DecodeAddress(addr string, defaultNet *chaincfg.Params) (lbcutil.Address, error) {
+func DecodeAddress(addr string, defaultNet *chaincfg.Params) (btcutil.Address, error) {
 	// Serialized public keys are either 65 bytes (130 hex chars) if
 	// uncompressed/hybrid or 33 bytes (66 hex chars) if compressed.
 	if len(addr) == 130 || len(addr) == 66 {
@@ -26,16 +25,16 @@ func DecodeAddress(addr string, defaultNet *chaincfg.Params) (lbcutil.Address, e
 		if err != nil {
 			return nil, err
 		}
-		return lbcutil.NewAddressPubKey(serializedPubKey, defaultNet)
+		return btcutil.NewAddressPubKey(serializedPubKey, defaultNet)
 	}
 
 	// Switch on decoded length to determine the type.
 	decoded, netID, err := base58.CheckDecode(addr)
 	if err != nil {
 		if err == base58.ErrChecksum {
-			return nil, lbcutil.ErrChecksumMismatch
+			return nil, btcutil.ErrChecksumMismatch
 		}
-		return nil, errors.WithStack(errors.Newf("decoded address[%s] is of unknown format even with default chainparams[%s]", addr, defaultNet.Name))
+		return nil, errors.Err("decoded address[%s] is of unknown format even with default chainparams[%s]", addr, defaultNet.Name)
 	}
 
 	switch len(decoded) {
@@ -44,16 +43,16 @@ func DecodeAddress(addr string, defaultNet *chaincfg.Params) (lbcutil.Address, e
 		isP2SH := chaincfg.IsScriptHashAddrID(netID)
 		switch hash160 := decoded; {
 		case isP2PKH && isP2SH:
-			return nil, lbcutil.ErrAddressCollision
+			return nil, btcutil.ErrAddressCollision
 		case isP2PKH:
-			return lbcutil.NewAddressPubKeyHash(hash160, &chaincfg.Params{PubKeyHashAddrID: netID})
+			return btcutil.NewAddressPubKeyHash(hash160, &chaincfg.Params{PubKeyHashAddrID: netID})
 		case isP2SH:
-			return lbcutil.NewAddressScriptHashFromHash(hash160, &chaincfg.Params{ScriptHashAddrID: netID})
+			return btcutil.NewAddressScriptHashFromHash(hash160, &chaincfg.Params{ScriptHashAddrID: netID})
 		default:
-			return nil, lbcutil.ErrUnknownAddressType
+			return nil, btcutil.ErrUnknownAddressType
 		}
 
 	default:
-		return nil, errors.WithStack(errors.New("decoded address is of unknown size"))
+		return nil, errors.Err("decoded address is of unknown size")
 	}
 }

lbrycrd/address_test.go

@@ -8,7 +8,5 @@ func TestDecodeAddress(t *testing.T) {
 	if err != nil {
 		t.Error(err)
 	}
-	if btcAddr.EncodeAddress() != addr {
+	println(btcAddr.EncodeAddress())
-		t.Errorf("expected: %s, actual: %s", addr, btcAddr.EncodeAddress())
-	}
 }

lbrycrd/blockchain_test.go

@@ -3,7 +3,7 @@ package lbrycrd_test
 import (
 	"testing"
 
-	"github.com/lbryio/lbry.go/v3/lbrycrd"
+	"github.com/lbryio/lbry.go/v2/lbrycrd"
 )
 
 var claimIdTests = []struct {

lbrycrd/channel.go

@@ -1,16 +1,14 @@
 package lbrycrd
 
 import (
-	"github.com/lbryio/lbry.go/v3/schema/keys"
+	"github.com/btcsuite/btcd/btcec"
-	c "github.com/lbryio/lbry.go/v3/schema/stake"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
-
+	"github.com/lbryio/lbry.go/v2/schema/keys"
-	"github.com/lbryio/lbcd/btcec"
+	c "github.com/lbryio/lbry.go/v2/schema/stake"
 	pb "github.com/lbryio/types/v2/go"
-
-	"github.com/cockroachdb/errors"
 )
 
-func NewChannel() (*c.Helper, *btcec.PrivateKey, error) {
+func NewChannel() (*c.StakeHelper, *btcec.PrivateKey, error) {
 	claimChannel := new(pb.Claim_Channel)
 	channel := new(pb.Channel)
 	claimChannel.Channel = channel
@@ -20,14 +18,14 @@ func NewChannel() (*c.Helper, *btcec.PrivateKey, error) {
 
 	privateKey, err := btcec.NewPrivateKey(btcec.S256())
 	if err != nil {
-		return nil, nil, errors.WithStack(err)
+		return nil, nil, errors.Err(err)
 	}
 	pubkeyBytes, err := keys.PublicKeyToDER(privateKey.PubKey())
 	if err != nil {
-		return nil, nil, errors.WithStack(err)
+		return nil, nil, errors.Err(err)
 	}
 
-	helper := c.Helper{Claim: pbClaim}
+	helper := c.StakeHelper{Claim: pbClaim}
 	helper.Version = c.NoSig
 	helper.Claim.GetChannel().PublicKey = pubkeyBytes
 	helper.Claim.Tags = []string{}

lbrycrd/claim.go

@@ -3,15 +3,15 @@ package lbrycrd
 import (
 	"encoding/hex"
 
-	c "github.com/lbryio/lbry.go/v3/schema/stake"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
+	c "github.com/lbryio/lbry.go/v2/schema/stake"
 	pb "github.com/lbryio/types/v2/go"
 
-	"github.com/cockroachdb/errors"
+	"github.com/btcsuite/btcd/btcec"
-	"github.com/lbryio/lbcd/btcec"
+	"github.com/btcsuite/btcd/wire"
-	"github.com/lbryio/lbcd/wire"
 )
 
-func NewImageStreamClaim() (*c.Helper, error) {
+func NewImageStreamClaim() (*c.StakeHelper, error) {
 	streamClaim := new(pb.Claim_Stream)
 	stream := new(pb.Stream)
 	image := new(pb.Stream_Image)
@@ -23,12 +23,12 @@ func NewImageStreamClaim() (*c.Helper, error) {
 	pbClaim := new(pb.Claim)
 	pbClaim.Type = streamClaim
 
-	helper := c.Helper{Claim: pbClaim}
+	helper := c.StakeHelper{Claim: pbClaim}
 
 	return &helper, nil
 }
 
-func NewVideoStreamClaim() (*c.Helper, error) {
+func NewVideoStreamClaim() (*c.StakeHelper, error) {
 	streamClaim := new(pb.Claim_Stream)
 	stream := new(pb.Stream)
 	video := new(pb.Stream_Video)
@@ -39,12 +39,12 @@ func NewVideoStreamClaim() (*c.Helper, error) {
 	pbClaim := new(pb.Claim)
 	pbClaim.Type = streamClaim
 
-	helper := c.Helper{Claim: pbClaim}
+	helper := c.StakeHelper{Claim: pbClaim}
 
 	return &helper, nil
 }
 
-func NewStreamClaim(title, description string) (*c.Helper, error) {
+func NewStreamClaim(title, description string) (*c.StakeHelper, error) {
 	streamClaim := new(pb.Claim_Stream)
 	stream := new(pb.Stream)
 	streamClaim.Stream = stream
@@ -52,17 +52,17 @@ func NewStreamClaim(title, description string) (*c.Helper, error) {
 	pbClaim := new(pb.Claim)
 	pbClaim.Type = streamClaim
 
-	helper := c.Helper{Claim: pbClaim}
+	helper := c.StakeHelper{Claim: pbClaim}
 	helper.Claim.Title = title
 	helper.Claim.Description = description
 
 	return &helper, nil
 }
 
-func SignClaim(rawTx *wire.MsgTx, privKey btcec.PrivateKey, claim, channel *c.Helper, channelClaimID string) error {
+func SignClaim(rawTx *wire.MsgTx, privKey btcec.PrivateKey, claim, channel *c.StakeHelper, channelClaimID string) error {
 	claimIDHexBytes, err := hex.DecodeString(channelClaimID)
 	if err != nil {
-		return errors.WithStack(err)
+		return errors.Err(err)
 	}
 	claim.Version = c.WithSig
 	claim.ClaimID = rev(claimIDHexBytes)