Merge pull request #10 from lbryio/eagerloading_depth

Fix multi-depth eager loading of relationships. If a relationship is …

boilingcore/imports.go

@@ -170,8 +170,8 @@ func newImporter() importer {
 			`"time"`,
 		},
 		thirdParty: importList{
-			`"github.com/lbryio/lbry.go/extras/errors"`,
-			`"github.com/lbryio/lbry.go/extras/null"`,
+			`"github.com/lbryio/lbry.go/v2/extras/errors"`,
+			`"github.com/lbryio/lbry.go/v2/extras/null"`,
 			`"github.com/lbryio/sqlboiler/boil"`,
 			`"github.com/lbryio/sqlboiler/queries"`,
 			`"github.com/lbryio/sqlboiler/queries/qm"`,
@@ -183,9 +183,10 @@ func newImporter() importer {
 		"boil_queries": imports{
 			standard: importList{
 				`"fmt"`,
+				`"strings"`,
 			},
 			thirdParty: importList{
-				`"github.com/lbryio/lbry.go/extras/errors"`,
+				`"github.com/lbryio/lbry.go/v2/extras/errors"`,
 				`"github.com/lbryio/sqlboiler/boil"`,
 				`"github.com/lbryio/sqlboiler/queries"`,
 				`"github.com/lbryio/sqlboiler/queries/qm"`,
@@ -194,7 +195,7 @@ func newImporter() importer {
 		},
 		"boil_types": {
 			thirdParty: importList{
-				`"github.com/lbryio/lbry.go/extras/errors"`,
+				`"github.com/lbryio/lbry.go/v2/extras/errors"`,
 				`"github.com/lbryio/sqlboiler/strmangle"`,
 			},
 		},
@@ -227,7 +228,7 @@ func newImporter() importer {
 			},
 			thirdParty: importList{
 				`"github.com/kat-co/vala"`,
-				`"github.com/lbryio/lbry.go/extras/errors"`,
+				`"github.com/lbryio/lbry.go/v2/extras/errors"`,
 				`"github.com/lbryio/sqlboiler/boil"`,
 				`"github.com/spf13/viper"`,
 			},
@@ -265,7 +266,7 @@ func newImporter() importer {
 				`"strings"`,
 			},
 			thirdParty: importList{
-				`"github.com/lbryio/lbry.go/extras/errors"`,
+				`"github.com/lbryio/lbry.go/v2/extras/errors"`,
 				`"github.com/lbryio/sqlboiler/bdb/drivers"`,
 				`"github.com/lbryio/sqlboiler/randomize"`,
 				`_ "github.com/lib/pq"`,
@@ -285,7 +286,7 @@ func newImporter() importer {
 			},
 			thirdParty: importList{
 				`_ "github.com/go-sql-driver/mysql"`,
-				`"github.com/lbryio/lbry.go/extras/errors"`,
+				`"github.com/lbryio/lbry.go/v2/extras/errors"`,
 				`"github.com/lbryio/sqlboiler/bdb/drivers"`,
 				`"github.com/lbryio/sqlboiler/randomize"`,
 				`"github.com/spf13/viper"`,
@@ -302,7 +303,7 @@ func newImporter() importer {
 			},
 			thirdParty: importList{
 				`_ "github.com/denisenkom/go-mssqldb"`,
-				`"github.com/lbryio/lbry.go/extras/errors"`,
+				`"github.com/lbryio/lbry.go/v2/extras/errors"`,
 				`"github.com/lbryio/sqlboiler/bdb/drivers"`,
 				`"github.com/lbryio/sqlboiler/randomize"`,
 				`"github.com/spf13/viper"`,
@@ -315,55 +316,55 @@ func newImporter() importer {
 	// TranslateColumnType to see the type assignments.
 	imp.BasedOnType = mapImports{
 		"null.Float32": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Float64": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Int": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Int8": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Int16": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Int32": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Int64": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Uint": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Uint8": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Uint16": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Uint32": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Uint64": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.String": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Bool": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Time": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.JSON": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"null.Bytes": {
-			thirdParty: importList{`"github.com/lbryio/lbry.go/extras/null"`},
+			thirdParty: importList{`"github.com/lbryio/lbry.go/v2/extras/null"`},
 		},
 		"time.Time": {
 			standard: importList{`"time"`},

boilingcore/imports_test.go

@@ -246,7 +246,7 @@ func TestCombineTypeImports(t *testing.T) {
 		},
 		thirdParty: importList{
 			`"github.com/lbryio/sqlboiler/boil"`,
-			`"github.com/lbryio/lbry.go/extras/null"`,
+			`"github.com/lbryio/lbry.go/v2/extras/null"`,
 		},
 	}
 
@@ -281,7 +281,7 @@ func TestCombineTypeImports(t *testing.T) {
 		},
 		thirdParty: importList{
 			`"github.com/lbryio/sqlboiler/boil"`,
-			`"github.com/lbryio/lbry.go/extras/null"`,
+			`"github.com/lbryio/lbry.go/v2/extras/null"`,
 		},
 	}
 
@@ -297,7 +297,7 @@ func TestCombineImports(t *testing.T) {
 
 	a := imports{
 		standard:   importList{"fmt"},
-		thirdParty: importList{"github.com/lbryio/sqlboiler", "github.com/lbryio/lbry.go/extras/null"},
+		thirdParty: importList{"github.com/lbryio/sqlboiler", "github.com/lbryio/lbry.go/v2/extras/null"},
 	}
 	b := imports{
 		standard:   importList{"os"},
@@ -309,8 +309,8 @@ func TestCombineImports(t *testing.T) {
 	if c.standard[0] != "fmt" && c.standard[1] != "os" {
 		t.Errorf("Wanted: fmt, os got: %#v", c.standard)
 	}
-	if c.thirdParty[0] != "github.com/lbryio/sqlboiler" && c.thirdParty[1] != "github.com/lbryio/lbry.go/extras/null" {
-		t.Errorf("Wanted: github.com/lbryio/sqlboiler, github.com/lbryio/lbry.go/extras/null got: %#v", c.thirdParty)
+	if c.thirdParty[0] != "github.com/lbryio/sqlboiler" && c.thirdParty[1] != "github.com/lbryio/lbry.go/v2/extras/null" {
+		t.Errorf("Wanted: github.com/lbryio/sqlboiler, github.com/lbryio/lbry.go/v2/extras/null got: %#v", c.thirdParty)
 	}
 }
 

queries/eager_load.go

@@ -5,9 +5,9 @@ import (
 	"reflect"
 	"strings"
 
-	"github.com/pkg/errors"
 	"github.com/lbryio/sqlboiler/boil"
 	"github.com/lbryio/sqlboiler/strmangle"
+	"github.com/pkg/errors"
 )
 
 type loadRelationshipState struct {
@@ -206,9 +206,16 @@ func (l loadRelationshipState) loadRelationshipsRecurse(depth int, obj reflect.V
 	}
 
 	bkind := kindStruct
-	if reflect.Indirect(loadedObject).Kind() != reflect.Struct {
+	if derefed := reflect.Indirect(loadedObject); derefed.Kind() != reflect.Struct {
 		bkind = kindPtrSliceStruct
-		loadedObject = loadedObject.Addr()
+
+		// Convert away any helper slice types
+		// elemType is *elem (from []*elem or helperSliceType)
+		// sliceType is *[]*elem
+		elemType := derefed.Type().Elem()
+		sliceType := reflect.PtrTo(reflect.SliceOf(elemType))
+
+		loadedObject = loadedObject.Addr().Convert(sliceType)
 	}
 	return l.loadRelationships(depth+1, loadedObject.Interface(), bkind)
 }
@@ -241,6 +248,9 @@ func collectLoaded(key string, loadingFrom reflect.Value) (reflect.Value, bindKi
 	if loadedType.Elem().Kind() == reflect.Struct {
 		bkind = kindStruct
 		loadedType = reflect.SliceOf(loadedType)
+	} else {
+		// Ensure that we get rid of all the helper "XSlice" types
+		loadedType = reflect.SliceOf(loadedType.Elem())
 	}
 
 	collection := reflect.MakeSlice(loadedType, 0, 0)
@@ -249,9 +259,13 @@ func collectLoaded(key string, loadingFrom reflect.Value) (reflect.Value, bindKi
 	for {
 		switch bkind {
 		case kindStruct:
-			collection = reflect.Append(collection, loadedObject)
+			if !loadedObject.IsNil() {
+				collection = reflect.Append(collection, loadedObject)
+			}
 		case kindPtrSliceStruct:
-			collection = reflect.AppendSlice(collection, loadedObject)
+			if !loadedObject.IsNil() {
+				collection = reflect.AppendSlice(collection, loadedObject)
+			}
 		}
 
 		i++

queries/helpers_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	null "github.com/lbryio/lbry.go/extras/null"
+	null "github.com/lbryio/lbry.go/v2/extras/null"
 )
 
 type testObj struct {

queries/qm/query_mods.go

@@ -8,7 +8,9 @@ type QueryMod func(q *queries.Query)
 // Apply the query mods to the Query object
 func Apply(q *queries.Query, mods ...QueryMod) {
 	for _, mod := range mods {
-		mod(q)
+		if mod != nil {
+			mod(q)
+		}
 	}
 }
 
@@ -123,6 +125,12 @@ func From(from string) QueryMod {
 	}
 }
 
+func ForceIndex(index string) QueryMod {
+	return func(q *queries.Query) {
+		queries.SetForceIndex(q, index)
+	}
+}
+
 // Limit the number of returned rows
 func Limit(limit int) QueryMod {
 	return func(q *queries.Query) {

queries/query.go

@@ -4,7 +4,7 @@ import (
 	"database/sql"
 	"fmt"
 
-	"github.com/lbryio/lbry.go/extras/errors"
+	"github.com/lbryio/lbry.go/v2/extras/errors"
 	"github.com/lbryio/sqlboiler/boil"
 )
 
@@ -30,6 +30,7 @@ type Query struct {
 	selectCols []string
 	count      bool
 	from       []string
+	forceindex string
 	joins      []join
 	where      []where
 	in         []in
@@ -263,6 +264,11 @@ func SetLastWhereAsOr(q *Query) {
 	q.where[len(q.where)-1].orSeparator = true
 }
 
+// SetForceIndex sets the index to be used by the query
+func SetForceIndex(q *Query, index string){
+	q.forceindex = index
+}
+
 // SetLastInAsOr sets the or separator for the tail "IN" in the slice
 func SetLastInAsOr(q *Query) {
 	if len(q.in) == 0 {

queries/query_builders.go

@@ -76,7 +76,13 @@ func buildSelectQuery(q *Query) (*bytes.Buffer, []interface{}) {
 		buf.WriteByte(')')
 	}
 
-	fmt.Fprintf(buf, " FROM %s", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "))
+	if len(q.forceindex) > 0 {
+		fmt.Fprintf(buf, " FROM %s FORCE INDEX (%s)", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "),q.forceindex)
+
+	}else{
+		fmt.Fprintf(buf, " FROM %s", strings.Join(strmangle.IdentQuoteSlice(q.dialect.LQ, q.dialect.RQ, q.from), ", "))
+
+	}
 
 	if len(q.joins) > 0 {
 		argsLen := len(args)

randomize/randomize.go

@@ -14,7 +14,7 @@ import (
 	"sync/atomic"
 	"time"
 
-	null "github.com/lbryio/lbry.go/extras/null"
+	null "github.com/lbryio/lbry.go/v2/extras/null"
 
 	"github.com/pkg/errors"
 	"github.com/satori/go.uuid"

randomize/randomize_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 	"time"
 
-	null "github.com/lbryio/lbry.go/extras/null"
+	null "github.com/lbryio/lbry.go/v2/extras/null"
 )
 
 func TestRandomizeStruct(t *testing.T) {

templates/singleton/boil_queries.tpl

@@ -27,11 +27,15 @@ func mergeModels(tx boil.Executor, primaryID uint64, secondaryID uint64, foreign
 	var err error
 
 	for _, conflict := range conflictingKeys {
-		err = deleteConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
-		if err != nil {
-			return err
-		}
-	}
+        if len(conflict.columns) == 1 && conflict.columns[0] == conflict.objectIdColumn {
+            err = deleteOneToOneConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
+        } else {
+            err = deleteOneToManyConflictsBeforeMerge(tx, conflict, primaryID, secondaryID)
+        }
+        if err != nil {
+            return err
+        }
+     }
 
 	for _, fk := range foreignKeys {
 		// TODO: use NewQuery here, not plain sql
@@ -48,53 +52,97 @@ func mergeModels(tx boil.Executor, primaryID uint64, secondaryID uint64, foreign
 	return checkMerge(tx, foreignKeys)
 }
 
-func deleteConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
-	conflictingColumns := strmangle.SetComplement(conflict.columns, []string{conflict.objectIdColumn})
-
-	if len(conflictingColumns) < 1 {
-		return nil
-	} else if len(conflictingColumns) > 1 {
-		return errors.Err("this doesnt work for unique keys with more than two columns (yet)")
-	}
-
+func deleteOneToOneConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
 	query := fmt.Sprintf(
-		"SELECT %s FROM %s WHERE %s IN (%s) GROUP BY %s HAVING count(distinct %s) > 1",
-		conflictingColumns[0], conflict.table, conflict.objectIdColumn,
+		"SELECT COUNT(*) FROM %s WHERE %s IN (%s)",
+		conflict.table, conflict.objectIdColumn,
 		strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1),
-		conflictingColumns[0], conflict.objectIdColumn,
 	)
 
-	rows, err := tx.Query(query, primaryID, secondaryID)
-	defer rows.Close()
+	var count int
+	err := tx.QueryRow(query, primaryID, secondaryID).Scan(&count)
 	if err != nil {
 		return errors.Err(err)
 	}
 
-	args := []interface{}{secondaryID}
-	for rows.Next() {
-		var value string
-		err = rows.Scan(&value)
-		if err != nil {
-			return errors.Err(err)
-		}
-		args = append(args, value)
-	}
-
-	// if no rows found, no need to delete anything
-	if len(args) < 2 {
-		return nil
+	if count > 2 {
+		return errors.Err("it should not be possible to have more than two rows here")
+	} else if count != 2 {
+		return nil // no conflicting rows
 	}
 
 	query = fmt.Sprintf(
-		"DELETE FROM %s WHERE %s = %s AND %s IN (%s)",
+		"DELETE FROM %s WHERE %s = %s",
 		conflict.table, conflict.objectIdColumn, strmangle.Placeholders(dialect.IndexPlaceholders, 1, 1, 1),
-		conflictingColumns[0], strmangle.Placeholders(dialect.IndexPlaceholders, len(args)-1, 2, 1),
 	)
 
-	_, err = tx.Exec(query, args...)
+	_, err = tx.Exec(query, secondaryID)
+	return errors.Err(err)
+}
+
+func deleteOneToManyConflictsBeforeMerge(tx boil.Executor, conflict conflictingUniqueKey, primaryID uint64, secondaryID uint64) error {
+	conflictingColumns := strmangle.SetComplement(conflict.columns, []string{conflict.objectIdColumn})
+
+	query := fmt.Sprintf(
+		"SELECT %s FROM %s WHERE %s IN (%s) GROUP BY %s HAVING count(distinct %s) > 1",
+		strings.Join(conflictingColumns, ","), conflict.table, conflict.objectIdColumn,
+		strmangle.Placeholders(dialect.IndexPlaceholders, 2, 1, 1),
+		strings.Join(conflictingColumns, ","), conflict.objectIdColumn,
+	)
+
+	//The selectParams should be the ObjectIDs to search for regarding the conflict.
+	rows, err := tx.Query(query, primaryID, secondaryID)
 	if err != nil {
 		return errors.Err(err)
 	}
+
+	//Since we don't don't know if advance how many columns the query returns, we have dynamically assign them to be
+	// used in the delete query.
+	colNames, err := rows.Columns()
+	if err != nil {
+		return errors.Err(err)
+	}
+	//Each row result of the query needs to be removed for being a conflicting row. Store each row's keys in an array.
+	var rowsToRemove = [][]interface{}(nil)
+	for rows.Next() {
+		//Set pointers for dynamic scan
+		iColPtrs := make([]interface{}, len(colNames))
+		for i := 0; i < len(colNames); i++ {
+			s := string("")
+			iColPtrs[i] = &s
+		}
+		//Dynamically scan n columns
+		err = rows.Scan(iColPtrs...)
+		if err != nil {
+			return errors.Err(err)
+		}
+		//Grab scanned values for query arguments
+		iCol := make([]interface{}, len(colNames))
+		for i, col := range iColPtrs {
+			x := col.(*string)
+			iCol[i] = *x
+		}
+		rowsToRemove = append(rowsToRemove, iCol)
+	}
+	defer rows.Close()
+
+	//This query will adjust dynamically depending on the number of conflicting keys, adding AND expressions for each
+	// key to ensure the right conflicting rows are deleted.
+	query = fmt.Sprintf(
+		"DELETE FROM %s %s",
+		conflict.table,
+		"WHERE "+strings.Join(conflict.columns, " = ? AND ")+" = ?",
+	)
+
+	//There could be multiple conflicting rows between ObjectIDs. In the SELECT query we grab each row and their column
+	// keys to be deleted here in a loop.
+	for _, rowToDelete := range rowsToRemove {
+		rowToDelete = append(rowToDelete, secondaryID)
+		_, err = tx.Exec(query, rowToDelete...)
+		if err != nil {
+			return errors.Err(err)
+		}
+	}
 	return nil
 }