Compare commits

..

No commits in common. "master" and "lbrygo_update" have entirely different histories.

10 changed files with 71 additions and 154 deletions

View file

@ -170,8 +170,8 @@ func newImporter() importer {
`"time"`,
},
thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/v2/extras/null"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/null"`,
`"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/sqlboiler/queries"`,
`"github.com/lbryio/sqlboiler/queries/qm"`,
@ -183,10 +183,9 @@ func newImporter() importer {
"boil_queries": imports{
standard: importList{
`"fmt"`,
`"strings"`,
},
thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/sqlboiler/queries"`,
`"github.com/lbryio/sqlboiler/queries/qm"`,
@ -195,7 +194,7 @@ func newImporter() importer {
},
"boil_types": {
thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/strmangle"`,
},
},
@ -228,7 +227,7 @@ func newImporter() importer {
},
thirdParty: importList{
`"github.com/kat-co/vala"`,
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/boil"`,
`"github.com/spf13/viper"`,
},
@ -266,7 +265,7 @@ func newImporter() importer {
`"strings"`,
},
thirdParty: importList{
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/bdb/drivers"`,
`"github.com/lbryio/sqlboiler/randomize"`,
`_ "github.com/lib/pq"`,
@ -286,7 +285,7 @@ func newImporter() importer {
},
thirdParty: importList{
`_ "github.com/go-sql-driver/mysql"`,
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/bdb/drivers"`,
`"github.com/lbryio/sqlboiler/randomize"`,
`"github.com/spf13/viper"`,
@ -303,7 +302,7 @@ func newImporter() importer {
},
thirdParty: importList{
`_ "github.com/denisenkom/go-mssqldb"`,
`"github.com/lbryio/lbry.go/v2/extras/errors"`,
`"github.com/lbryio/lbry.go/extras/errors"`,
`"github.com/lbryio/sqlboiler/bdb/drivers"`,
`"github.com/lbryio/sqlboiler/randomize"`,
`"github.com/spf13/viper"`,
@ -316,55 +315,55 @@ func newImporter() importer {
// TranslateColumnType to see the type assignments.
imp.BasedOnType = mapImports{
"null.Float32": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Float64": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Int": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Int8": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Int16": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Int32": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Int64": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Uint": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Uint8": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Uint16": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Uint32": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Uint64": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.String": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Bool": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Time": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.JSON": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"null.Bytes": {
thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
},
"time.Time": {
standard: importList{`"time"`},

View file

@ -246,7 +246,7 @@ func TestCombineTypeImports(t *testing.T) {
},
thirdParty: importList{
`"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/lbry.go/v2/extras/null"`,
`"github.com/lbryio/lbry.go/extras/null"`,
},
}
@ -281,7 +281,7 @@ func TestCombineTypeImports(t *testing.T) {
},
thirdParty: importList{
`"github.com/lbryio/sqlboiler/boil"`,
`"github.com/lbryio/lbry.go/v2/extras/null"`,
`"github.com/lbryio/lbry.go/extras/null"`,
},
}
@ -297,7 +297,7 @@ func TestCombineImports(t *testing.T) {
a := imports{
standard: importList{"fmt"},
thirdParty: importList{"github.com/lbryio/sqlboiler", "github.com/lbryio/lbry.go/v2/extras/null"},
thirdParty: importList{"github.com/lbryio/sqlboiler", "github.com/lbryio/lbry.go/extras/null"},
}
b := imports{
standard: importList{"os"},
@ -309,8 +309,8 @@ func TestCombineImports(t *testing.T) {
if c.standard[0] != "fmt" && c.standard[1] != "os" {
t.Errorf("Wanted: fmt, os got: %#v", c.standard)
}
if c.thirdParty[0] != "github.com/lbryio/sqlboiler" && c.thirdParty[1] != "github.com/lbryio/lbry.go/v2/extras/null" {
t.Errorf("Wanted: github.com/lbryio/sqlboiler, github.com/lbryio/lbry.go/v2/extras/null got: %#v", c.thirdParty)
if c.thirdParty[0] != "github.com/lbryio/sqlboiler" && c.thirdParty[1] != "github.com/lbryio/lbry.go/extras/null" {
t.Errorf("Wanted: github.com/lbryio/sqlboiler, github.com/lbryio/lbry.go/extras/null got: %#v", c.thirdParty)
}
}

View file

@ -5,9 +5,9 @@ import (
"reflect"
"strings"
"github.com/pkg/errors"
"github.com/lbryio/sqlboiler/boil"
"github.com/lbryio/sqlboiler/strmangle"
"github.com/pkg/errors"
)
type loadRelationshipState struct {
@ -206,16 +206,9 @@ func (l loadRelationshipState) loadRelationshipsRecurse(depth int, obj reflect.V
}
bkind := kindStruct
if derefed := reflect.Indirect(loadedObject); derefed.Kind() != reflect.Struct {
if reflect.Indirect(loadedObject).Kind() != reflect.Struct {
bkind = kindPtrSliceStruct
// Convert away any helper slice types
// elemType is *elem (from []*elem or helperSliceType)
// sliceType is *[]*elem
elemType := derefed.Type().Elem()
sliceType := reflect.PtrTo(reflect.SliceOf(elemType))
loadedObject = loadedObject.Addr().Convert(sliceType)
loadedObject = loadedObject.Addr()
}
return l.loadRelationships(depth+1, loadedObject.Interface(), bkind)
}
@ -248,9 +241,6 @@ func collectLoaded(key string, loadingFrom reflect.Value) (reflect.Value, bindKi
if loadedType.Elem().Kind() == reflect.Struct {
bkind = kindStruct
loadedType = reflect.SliceOf(loadedType)
} else {
// Ensure that we get rid of all the helper "XSlice" types
loadedType = reflect.SliceOf(loadedType.Elem())
}
collection := reflect.MakeSlice(loadedType, 0, 0)
@ -259,13 +249,9 @@ func collectLoaded(key string, loadingFrom reflect.Value) (reflect.Value, bindKi
for {
switch bkind {
case kindStruct:
if !loadedObject.IsNil() {
collection = reflect.Append(collection, loadedObject)
}
collection = reflect.Append(collection, loadedObject)
case kindPtrSliceStruct:
if !loadedObject.IsNil() {
collection = reflect.AppendSlice(collection, loadedObject)
}
collection = reflect.AppendSlice(collection, loadedObject)
}
i++

View file

@ -5,7 +5,7 @@ import (
"testing"
"time"
null "github.com/lbryio/lbry.go/v2/extras/null"
null "github.com/lbryio/lbry.go/extras/null"
)
type testObj struct {

View file

@ -8,9 +8,7 @@ type QueryMod func(q *queries.Query)
// Apply the query mods to the Query object
func Apply(q *queries.Query, mods ...QueryMod) {
for _, mod := range mods {
if mod != nil {
mod(q)
}
mod(q)
}
}
@ -125,12 +123,6 @@ func From(from string) QueryMod {
}
}
func ForceIndex(index string) QueryMod {
return func(q *queries.Query) {
queries.SetForceIndex(q, index)
}
}
// Limit the number of returned rows
func Limit(limit int) QueryMod {
return func(q *queries.Query) {

View file

@ -4,7 +4,7 @@ import (
"database/sql"
"fmt"
"github.com/lbryio/lbry.go/v2/extras/errors"
"github.com/lbryio/lbry.go/extras/errors"
"github.com/lbryio/sqlboiler/boil"
)
@ -30,7 +30,6 @@ type Query struct {
selectCols []string
count bool
from []string
forceindex string
joins []join
where []where
in []in
@ -264,11 +263,6 @@ func SetLastWhereAsOr(q *Query) {
q.where[len(q.where)-1].orSeparator = true
}
// SetForceIndex sets the index to be used by the query
func SetForceIndex(q *Query, index string){
q.forceindex = index
}
// SetLastInAsOr sets the or separator for the tail "IN" in the slice
func SetLastInAsOr(q *Query) {
if len(q.in) == 0 {

View file

@ -76,13 +76,7 @@ func buildSelectQuery(q *Query) (*bytes.Buffer, []interface{}) {
buf.WriteByte(')')
}
if len(q.forceindex) > 0 {
fmt.Fprintf(buf, " FROM %s FORCE INDEX (%s)", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "),q.forceindex)
}else{
fmt.Fprintf(buf, " FROM %s", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "))
}
fmt.Fprintf(buf, " FROM %s", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "))
if len(q.joins) > 0 {
argsLen := len(args)

View file

@ -14,7 +14,7 @@ import (
"sync/atomic"
"time"
null "github.com/lbryio/lbry.go/v2/extras/null"
null "github.com/lbryio/lbry.go/extras/null"
"github.com/pkg/errors"
"github.com/satori/go.uuid"

View file

@ -5,7 +5,7 @@ import (
"testing"
"time"
null "github.com/lbryio/lbry.go/v2/extras/null"
null "github.com/lbryio/lbry.go/extras/null"
)
func TestRandomizeStruct(t *testing.T) {

View file

@ -27,15 +27,11 @@ func mergeModels(tx boil.Executor, primaryID uint64, secondaryID uint64, foreign
var err error
for _, conflict := range conflictingKeys {
if len(conflict.columns) == 1 && conflict.columns[0] == conflict.objectIdColumn {
err = deleteOneToOneConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
} else {
err = deleteOneToManyConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
}
if err != nil {
return err
}
}
err = deleteConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
if err != nil {
return err
}
}
for _, fk := range foreignKeys {
// TODO: use NewQuery here, not plain sql
@ -52,96 +48,52 @@ func mergeModels(tx boil.Executor, primaryID uint64, secondaryID uint64, foreign
return checkMerge(tx, foreignKeys)
}
func deleteOneToOneConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
query := fmt.Sprintf(
"SELECT COUNT(*) FROM %s WHERE %s IN (%s)",
conflict.table, conflict.objectIdColumn,
strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1),
)
var count int
err := tx.QueryRow(query, primaryID, secondaryID).Scan(&count)
if err != nil {
return errors.Err(err)
}
if count > 2 {
return errors.Err("it should not be possible to have more than two rows here")
} else if count != 2 {
return nil // no conflicting rows
}
query = fmt.Sprintf(
"DELETE FROM %s WHERE %s = %s",
conflict.table, conflict.objectIdColumn, strmangle.Placeholders(dialect.IndexPlaceholders, 1, 1, 1),
)
_, err = tx.Exec(query, secondaryID)
return errors.Err(err)
}
func deleteOneToManyConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
func deleteConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
conflictingColumns := strmangle.SetComplement(conflict.columns, []string{conflict.objectIdColumn})
if len(conflictingColumns) < 1 {
return nil
} else if len(conflictingColumns) > 1 {
return errors.Err("this doesnt work for unique keys with more than two columns (yet)")
}
query := fmt.Sprintf(
"SELECT %s FROM %s WHERE %s IN (%s) GROUP BY %s HAVING count(distinct %s) > 1",
strings.Join(conflictingColumns, ","), conflict.table, conflict.objectIdColumn,
conflictingColumns[0], conflict.table, conflict.objectIdColumn,
strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1),
strings.Join(conflictingColumns, ","), conflict.objectIdColumn,
conflictingColumns[0], conflict.objectIdColumn,
)
//The selectParams should be the ObjectIDs to search for regarding the conflict.
rows, err := tx.Query(query, primaryID, secondaryID)
defer rows.Close()
if err != nil {
return errors.Err(err)
}
//Since we don't don't know if advance how many columns the query returns, we have dynamically assign them to be
// used in the delete query.
colNames, err := rows.Columns()
if err != nil {
return errors.Err(err)
}
//Each row result of the query needs to be removed for being a conflicting row. Store each row's keys in an array.
var rowsToRemove = [][]interface{}(nil)
args := []interface{}{secondaryID}
for rows.Next() {
//Set pointers for dynamic scan
iColPtrs := make([]interface{}, len(colNames))
for i := 0; i < len(colNames); i++ {
s := string("")
iColPtrs[i] = &s
}
//Dynamically scan n columns
err = rows.Scan(iColPtrs...)
var value string
err = rows.Scan(&value)
if err != nil {
return errors.Err(err)
}
//Grab scanned values for query arguments
iCol := make([]interface{}, len(colNames))
for i, col := range iColPtrs {
x := col.(*string)
iCol[i] = *x
}
rowsToRemove = append(rowsToRemove, iCol)
args = append(args, value)
}
// if no rows found, no need to delete anything
if len(args) < 2 {
return nil
}
defer rows.Close()
//This query will adjust dynamically depending on the number of conflicting keys, adding AND expressions for each
// key to ensure the right conflicting rows are deleted.
query = fmt.Sprintf(
"DELETE FROM %s %s",
conflict.table,
"WHERE "+strings.Join(conflict.columns, " = ? AND ")+" = ?",
"DELETE FROM %s WHERE %s = %s AND %s IN (%s)",
conflict.table, conflict.objectIdColumn, strmangle.Placeholders(dialect.IndexPlaceholders, 1, 1, 1),
conflictingColumns[0], strmangle.Placeholders(dialect.IndexPlaceholders, len(args)-1, 2, 1),
)
//There could be multiple conflicting rows between ObjectIDs. In the SELECT query we grab each row and their column
// keys to be deleted here in a loop.
for _, rowToDelete := range rowsToRemove {
rowToDelete = append(rowToDelete, secondaryID)
_, err = tx.Exec(query, rowToDelete...)
if err != nil {
return errors.Err(err)
}
_, err = tx.Exec(query, args...)
if err != nil {
return errors.Err(err)
}
return nil
}