Compare commits

..

No commits in common. "master" and "lbrygo_update" have entirely different histories.

10 changed files with 71 additions and 154 deletions

View file

@ -170,8 +170,8 @@ func newImporter() importer {
`"time"`, `"time"`,
}, },
thirdParty: importList{ thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/lbry.go/v2/extras/null"`, `"github.com/lbryio/lbry.go/extras/null"`,
`"github.com/lbryio/sqlboiler/boil"`, `"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/sqlboiler/queries"`, `"github.com/lbryio/sqlboiler/queries"`,
`"github.com/lbryio/sqlboiler/queries/qm"`, `"github.com/lbryio/sqlboiler/queries/qm"`,
@ -183,10 +183,9 @@ func newImporter() importer {
"boil_queries": imports{ "boil_queries": imports{
standard: importList{ standard: importList{
`"fmt"`, `"fmt"`,
`"strings"`,
}, },
thirdParty: importList{ thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/boil"`, `"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/sqlboiler/queries"`, `"github.com/lbryio/sqlboiler/queries"`,
`"github.com/lbryio/sqlboiler/queries/qm"`, `"github.com/lbryio/sqlboiler/queries/qm"`,
@ -195,7 +194,7 @@ func newImporter() importer {
}, },
"boil_types": { "boil_types": {
thirdParty: importList{ thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/strmangle"`, `"github.com/lbryio/sqlboiler/strmangle"`,
}, },
}, },
@ -228,7 +227,7 @@ func newImporter() importer {
}, },
thirdParty: importList{ thirdParty: importList{
`"github.com/kat-co/vala"`, `"github.com/kat-co/vala"`,
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/boil"`, `"github.com/lbryio/sqlboiler/boil"`,
`"github.com/spf13/viper"`, `"github.com/spf13/viper"`,
}, },
@ -266,7 +265,7 @@ func newImporter() importer {
`"strings"`, `"strings"`,
}, },
thirdParty: importList{ thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/bdb/drivers"`, `"github.com/lbryio/sqlboiler/bdb/drivers"`,
`"github.com/lbryio/sqlboiler/randomize"`, `"github.com/lbryio/sqlboiler/randomize"`,
`_ "github.com/lib/pq"`, `_ "github.com/lib/pq"`,
@ -286,7 +285,7 @@ func newImporter() importer {
}, },
thirdParty: importList{ thirdParty: importList{
`_ "github.com/go-sql-driver/mysql"`, `_ "github.com/go-sql-driver/mysql"`,
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/bdb/drivers"`, `"github.com/lbryio/sqlboiler/bdb/drivers"`,
`"github.com/lbryio/sqlboiler/randomize"`, `"github.com/lbryio/sqlboiler/randomize"`,
`"github.com/spf13/viper"`, `"github.com/spf13/viper"`,
@ -303,7 +302,7 @@ func newImporter() importer {
}, },
thirdParty: importList{ thirdParty: importList{
`_ "github.com/denisenkom/go-mssqldb"`, `_ "github.com/denisenkom/go-mssqldb"`,
`"github.com/lbryio/lbry.go/v2/extras/errors"`, `"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/bdb/drivers"`, `"github.com/lbryio/sqlboiler/bdb/drivers"`,
`"github.com/lbryio/sqlboiler/randomize"`, `"github.com/lbryio/sqlboiler/randomize"`,
`"github.com/spf13/viper"`, `"github.com/spf13/viper"`,
@ -316,55 +315,55 @@ func newImporter() importer {
// TranslateColumnType to see the type assignments. // TranslateColumnType to see the type assignments.
imp.BasedOnType = mapImports{ imp.BasedOnType = mapImports{
"null.Float32": { "null.Float32": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Float64": { "null.Float64": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Int": { "null.Int": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Int8": { "null.Int8": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Int16": { "null.Int16": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Int32": { "null.Int32": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Int64": { "null.Int64": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Uint": { "null.Uint": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Uint8": { "null.Uint8": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Uint16": { "null.Uint16": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Uint32": { "null.Uint32": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Uint64": { "null.Uint64": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.String": { "null.String": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Bool": { "null.Bool": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Time": { "null.Time": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.JSON": { "null.JSON": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"null.Bytes": { "null.Bytes": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`}, thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
}, },
"time.Time": { "time.Time": {
standard: importList{`"time"`}, standard: importList{`"time"`},

View file

@ -246,7 +246,7 @@ func TestCombineTypeImports(t *testing.T) {
}, },
thirdParty: importList{ thirdParty: importList{
`"github.com/lbryio/sqlboiler/boil"`, `"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/lbry.go/v2/extras/null"`, `"github.com/lbryio/lbry.go/extras/null"`,
}, },
} }
@ -281,7 +281,7 @@ func TestCombineTypeImports(t *testing.T) {
}, },
thirdParty: importList{ thirdParty: importList{
`"github.com/lbryio/sqlboiler/boil"`, `"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/lbry.go/v2/extras/null"`, `"github.com/lbryio/lbry.go/extras/null"`,
}, },
} }
@ -297,7 +297,7 @@ func TestCombineImports(t *testing.T) {
a := imports{ a := imports{
standard: importList{"fmt"}, standard: importList{"fmt"},
thirdParty: importList{"github.com/lbryio/sqlboiler", "github.com/lbryio/lbry.go/v2/extras/null"}, thirdParty: importList{"github.com/lbryio/sqlboiler", "github.com/lbryio/lbry.go/extras/null"},
} }
b := imports{ b := imports{
standard: importList{"os"}, standard: importList{"os"},
@ -309,8 +309,8 @@ func TestCombineImports(t *testing.T) {
if c.standard[0] != "fmt" && c.standard[1] != "os" { if c.standard[0] != "fmt" && c.standard[1] != "os" {
t.Errorf("Wanted: fmt, os got: %#v", c.standard) t.Errorf("Wanted: fmt, os got: %#v", c.standard)
} }
if c.thirdParty[0] != "github.com/lbryio/sqlboiler" && c.thirdParty[1] != "github.com/lbryio/lbry.go/v2/extras/null" { if c.thirdParty[0] != "github.com/lbryio/sqlboiler" && c.thirdParty[1] != "github.com/lbryio/lbry.go/extras/null" {
t.Errorf("Wanted: github.com/lbryio/sqlboiler, github.com/lbryio/lbry.go/v2/extras/null got: %#v", c.thirdParty) t.Errorf("Wanted: github.com/lbryio/sqlboiler, github.com/lbryio/lbry.go/extras/null got: %#v", c.thirdParty)
} }
} }

View file

@ -5,9 +5,9 @@ import (
"reflect" "reflect"
"strings" "strings"
"github.com/pkg/errors"
"github.com/lbryio/sqlboiler/boil" "github.com/lbryio/sqlboiler/boil"
"github.com/lbryio/sqlboiler/strmangle" "github.com/lbryio/sqlboiler/strmangle"
"github.com/pkg/errors"
) )
type loadRelationshipState struct { type loadRelationshipState struct {
@ -206,16 +206,9 @@ func (l loadRelationshipState) loadRelationshipsRecurse(depth int, obj reflect.V
} }
bkind := kindStruct bkind := kindStruct
if derefed := reflect.Indirect(loadedObject); derefed.Kind() != reflect.Struct { if reflect.Indirect(loadedObject).Kind() != reflect.Struct {
bkind = kindPtrSliceStruct bkind = kindPtrSliceStruct
loadedObject = loadedObject.Addr()
// Convert away any helper slice types
// elemType is *elem (from []*elem or helperSliceType)
// sliceType is *[]*elem
elemType := derefed.Type().Elem()
sliceType := reflect.PtrTo(reflect.SliceOf(elemType))
loadedObject = loadedObject.Addr().Convert(sliceType)
} }
return l.loadRelationships(depth+1, loadedObject.Interface(), bkind) return l.loadRelationships(depth+1, loadedObject.Interface(), bkind)
} }
@ -248,9 +241,6 @@ func collectLoaded(key string, loadingFrom reflect.Value) (reflect.Value, bindKi
if loadedType.Elem().Kind() == reflect.Struct { if loadedType.Elem().Kind() == reflect.Struct {
bkind = kindStruct bkind = kindStruct
loadedType = reflect.SliceOf(loadedType) loadedType = reflect.SliceOf(loadedType)
} else {
// Ensure that we get rid of all the helper "XSlice" types
loadedType = reflect.SliceOf(loadedType.Elem())
} }
collection := reflect.MakeSlice(loadedType, 0, 0) collection := reflect.MakeSlice(loadedType, 0, 0)
@ -259,14 +249,10 @@ func collectLoaded(key string, loadingFrom reflect.Value) (reflect.Value, bindKi
for { for {
switch bkind { switch bkind {
case kindStruct: case kindStruct:
if !loadedObject.IsNil() {
collection = reflect.Append(collection, loadedObject) collection = reflect.Append(collection, loadedObject)
}
case kindPtrSliceStruct: case kindPtrSliceStruct:
if !loadedObject.IsNil() {
collection = reflect.AppendSlice(collection, loadedObject) collection = reflect.AppendSlice(collection, loadedObject)
} }
}
i++ i++
if i >= lnFrom { if i >= lnFrom {

View file

@ -5,7 +5,7 @@ import (
"testing" "testing"
"time" "time"
null "github.com/lbryio/lbry.go/v2/extras/null" null "github.com/lbryio/lbry.go/extras/null"
) )
type testObj struct { type testObj struct {

View file

@ -8,10 +8,8 @@ type QueryMod func(q *queries.Query)
// Apply the query mods to the Query object // Apply the query mods to the Query object
func Apply(q *queries.Query, mods ...QueryMod) { func Apply(q *queries.Query, mods ...QueryMod) {
for _, mod := range mods { for _, mod := range mods {
if mod != nil {
mod(q) mod(q)
} }
}
} }
// SQL allows you to execute a plain SQL statement // SQL allows you to execute a plain SQL statement
@ -125,12 +123,6 @@ func From(from string) QueryMod {
} }
} }
func ForceIndex(index string) QueryMod {
return func(q *queries.Query) {
queries.SetForceIndex(q, index)
}
}
// Limit the number of returned rows // Limit the number of returned rows
func Limit(limit int) QueryMod { func Limit(limit int) QueryMod {
return func(q *queries.Query) { return func(q *queries.Query) {

View file

@ -4,7 +4,7 @@ import (
"database/sql" "database/sql"
"fmt" "fmt"
"github.com/lbryio/lbry.go/v2/extras/errors" "github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/sqlboiler/boil" "github.com/lbryio/sqlboiler/boil"
) )
@ -30,7 +30,6 @@ type Query struct {
selectCols []string selectCols []string
count bool count bool
from []string from []string
forceindex string
joins []join joins []join
where []where where []where
in []in in []in
@ -264,11 +263,6 @@ func SetLastWhereAsOr(q *Query) {
q.where[len(q.where)-1].orSeparator = true q.where[len(q.where)-1].orSeparator = true
} }
// SetForceIndex sets the index to be used by the query
func SetForceIndex(q *Query, index string){
q.forceindex = index
}
// SetLastInAsOr sets the or separator for the tail "IN" in the slice // SetLastInAsOr sets the or separator for the tail "IN" in the slice
func SetLastInAsOr(q *Query) { func SetLastInAsOr(q *Query) {
if len(q.in) == 0 { if len(q.in) == 0 {

View file

@ -76,14 +76,8 @@ func buildSelectQuery(q *Query) (*bytes.Buffer, []interface{}) {
buf.WriteByte(')') buf.WriteByte(')')
} }
if len(q.forceindex) > 0 {
fmt.Fprintf(buf, " FROM %s FORCE INDEX (%s)", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "),q.forceindex)
}else{
fmt.Fprintf(buf, " FROM %s", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", ")) fmt.Fprintf(buf, " FROM %s", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "))
}
if len(q.joins) > 0 { if len(q.joins) > 0 {
argsLen := len(args) argsLen := len(args)
joinBuf := strmangle.GetBuffer() joinBuf := strmangle.GetBuffer()

View file

@ -14,7 +14,7 @@ import (
"sync/atomic" "sync/atomic"
"time" "time"
null "github.com/lbryio/lbry.go/v2/extras/null" null "github.com/lbryio/lbry.go/extras/null"
"github.com/pkg/errors" "github.com/pkg/errors"
"github.com/satori/go.uuid" "github.com/satori/go.uuid"

View file

@ -5,7 +5,7 @@ import (
"testing" "testing"
"time" "time"
null "github.com/lbryio/lbry.go/v2/extras/null" null "github.com/lbryio/lbry.go/extras/null"
) )
func TestRandomizeStruct(t *testing.T) { func TestRandomizeStruct(t *testing.T) {

View file

@ -27,11 +27,7 @@ func mergeModels(tx boil.Executor, primaryID uint64, secondaryID uint64, foreign
var err error var err error
for _, conflict := range conflictingKeys { for _, conflict := range conflictingKeys {
if len(conflict.columns) == 1 && conflict.columns[0] == conflict.objectIdColumn { err = deleteConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
err = deleteOneToOneConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
} else {
err = deleteOneToManyConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
}
if err != nil { if err != nil {
return err return err
} }
@ -52,97 +48,53 @@ func mergeModels(tx boil.Executor, primaryID uint64, secondaryID uint64, foreign
return checkMerge(tx, foreignKeys) return checkMerge(tx, foreignKeys)
} }
func deleteOneToOneConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error { func deleteConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
query := fmt.Sprintf(
"SELECT COUNT(*) FROM %s WHERE %s IN (%s)",
conflict.table, conflict.objectIdColumn,
strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1),
)
var count int
err := tx.QueryRow(query, primaryID, secondaryID).Scan(&count)
if err != nil {
return errors.Err(err)
}
if count > 2 {
return errors.Err("it should not be possible to have more than two rows here")
} else if count != 2 {
return nil // no conflicting rows
}
query = fmt.Sprintf(
"DELETE FROM %s WHERE %s = %s",
conflict.table, conflict.objectIdColumn, strmangle.Placeholders(dialect.IndexPlaceholders, 1, 1, 1),
)
_, err = tx.Exec(query, secondaryID)
return errors.Err(err)
}
func deleteOneToManyConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
conflictingColumns := strmangle.SetComplement(conflict.columns, []string{conflict.objectIdColumn}) conflictingColumns := strmangle.SetComplement(conflict.columns, []string{conflict.objectIdColumn})
if len(conflictingColumns) < 1 {
return nil
} else if len(conflictingColumns) > 1 {
return errors.Err("this doesnt work for unique keys with more than two columns (yet)")
}
query := fmt.Sprintf( query := fmt.Sprintf(
"SELECT %s FROM %s WHERE %s IN (%s) GROUP BY %s HAVING count(distinct %s) > 1", "SELECT %s FROM %s WHERE %s IN (%s) GROUP BY %s HAVING count(distinct %s) > 1",
strings.Join(conflictingColumns, ","), conflict.table, conflict.objectIdColumn, conflictingColumns[0], conflict.table, conflict.objectIdColumn,
strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1), strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1),
strings.Join(conflictingColumns, ","), conflict.objectIdColumn, conflictingColumns[0], conflict.objectIdColumn,
) )
//The selectParams should be the ObjectIDs to search for regarding the conflict.
rows, err := tx.Query(query, primaryID, secondaryID) rows, err := tx.Query(query, primaryID, secondaryID)
if err != nil {
return errors.Err(err)
}
//Since we don't don't know if advance how many columns the query returns, we have dynamically assign them to be
// used in the delete query.
colNames, err := rows.Columns()
if err != nil {
return errors.Err(err)
}
//Each row result of the query needs to be removed for being a conflicting row. Store each row's keys in an array.
var rowsToRemove = [][]interface{}(nil)
for rows.Next() {
//Set pointers for dynamic scan
iColPtrs := make([]interface{}, len(colNames))
for i := 0; i < len(colNames); i++ {
s := string("")
iColPtrs[i] = &s
}
//Dynamically scan n columns
err = rows.Scan(iColPtrs...)
if err != nil {
return errors.Err(err)
}
//Grab scanned values for query arguments
iCol := make([]interface{}, len(colNames))
for i, col := range iColPtrs {
x := col.(*string)
iCol[i] = *x
}
rowsToRemove = append(rowsToRemove, iCol)
}
defer rows.Close() defer rows.Close()
if err != nil {
return errors.Err(err)
}
args := []interface{}{secondaryID}
for rows.Next() {
var value string
err = rows.Scan(&value)
if err != nil {
return errors.Err(err)
}
args = append(args, value)
}
// if no rows found, no need to delete anything
if len(args) < 2 {
return nil
}
//This query will adjust dynamically depending on the number of conflicting keys, adding AND expressions for each
// key to ensure the right conflicting rows are deleted.
query = fmt.Sprintf( query = fmt.Sprintf(
"DELETE FROM %s %s", "DELETE FROM %s WHERE %s = %s AND %s IN (%s)",
conflict.table, conflict.table, conflict.objectIdColumn, strmangle.Placeholders(dialect.IndexPlaceholders, 1, 1, 1),
"WHERE "+strings.Join(conflict.columns, " = ? AND ")+" = ?", conflictingColumns[0], strmangle.Placeholders(dialect.IndexPlaceholders, len(args)-1, 2, 1),
) )
//There could be multiple conflicting rows between ObjectIDs. In the SELECT query we grab each row and their column _, err = tx.Exec(query, args...)
// keys to be deleted here in a loop.
for _, rowToDelete := range rowsToRemove {
rowToDelete = append(rowToDelete, secondaryID)
_, err = tx.Exec(query, rowToDelete...)
if err != nil { if err != nil {
return errors.Err(err) return errors.Err(err)
} }
}
return nil return nil
} }