txscript: Convert reference tests to new format.

This updates the data driven transaction script tests to use the most
recent format and test data as implemented by Core so the test data can
more easily be updated and help prove cross-compatibility correctness.

In particular, the new format combines the previously separate valid and
invalid test data files into a single file and adds a field for the
expected result.  This is a nice improvement since it means tests can
now ensure script failures are due to a specific expected reason as
opposed to only generically detecting failure as the previous format
required.

The btcd script engine typically returns more fine grained errors than
the test data expects, so the test adapter handles this by allowing
expected errors in the test data to be mapped to multiple txscript
errors.

It should also be noted that the tests related to segwit have been
stripped from the data since the segwit PR has not landed in master yet,
however the test adapter does recognize the new ability for optional
segwit data to be supplied, though it will need to properly construct
the transaction using that data when the time comes.

txscript/reference_test.go

@@ -1,4 +1,4 @@
-// Copyright (c) 2013-2016 The btcsuite developers
+// Copyright (c) 2013-2017 The btcsuite developers
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
 
@@ -20,19 +20,31 @@ import (
 	"github.com/btcsuite/btcutil"
 )
 
-// testName returns a descriptive test name for the given reference test data.
+// scriptTestName returns a descriptive test name for the given reference script
-func testName(test []string) (string, error) {
+// test data.
-	var name string
+func scriptTestName(test []interface{}) (string, error) {
-
+	// Account for any optional leading witness data.
-	if len(test) < 3 || len(test) > 4 {
+	var witnessOffset int
-		return name, fmt.Errorf("invalid test length %d", len(test))
+	if _, ok := test[0].([]interface{}); ok {
+		witnessOffset++
 	}
 
-	if len(test) == 4 {
+	// In addition to the optional leading witness data, the test must
-		name = fmt.Sprintf("test (%s)", test[3])
+	// consist of at least a signature script, public key script, flags,
+	// and expected error.  Finally, it may optionally contain a comment.
+	if len(test) < witnessOffset+4 || len(test) > witnessOffset+5 {
+		return "", fmt.Errorf("invalid test length %d", len(test))
+	}
+
+	// Use the comment for the test name if one is specified, otherwise,
+	// construct the name based on the signature script, public key script,
+	// and flags.
+	var name string
+	if len(test) == witnessOffset+5 {
+		name = fmt.Sprintf("test (%s)", test[witnessOffset+4])
 	} else {
-		name = fmt.Sprintf("test ([%s, %s, %s])", test[0], test[1],
+		name = fmt.Sprintf("test ([%s, %s, %s])", test[witnessOffset],
-			test[2])
+			test[witnessOffset+1], test[witnessOffset+2])
 	}
 	return name, nil
 }
@@ -113,7 +125,7 @@ func parseShortForm(script string) ([]byte, error) {
 		} else if opcode, ok := shortFormOps[tok]; ok {
 			builder.AddOp(opcode)
 		} else {
-			return nil, fmt.Errorf("bad token \"%s\"", tok)
+			return nil, fmt.Errorf("bad token %q", tok)
 		}
 
 	}
@@ -161,6 +173,74 @@ func parseScriptFlags(flagStr string) (ScriptFlags, error) {
 	return flags, nil
 }
 
+// parseExpectedResult parses the provided expected result string into allowed
+// script error codes.  An error is returned if the expected result string is
+// not supported.
+func parseExpectedResult(expected string) ([]ErrorCode, error) {
+	switch expected {
+	case "OK":
+		return nil, nil
+	case "UNKNOWN_ERROR":
+		return []ErrorCode{ErrNumberTooBig, ErrMinimalData}, nil
+	case "PUBKEYTYPE":
+		return []ErrorCode{ErrPubKeyType}, nil
+	case "SIG_DER":
+		return []ErrorCode{ErrSigDER, ErrInvalidSigHashType}, nil
+	case "EVAL_FALSE":
+		return []ErrorCode{ErrEvalFalse, ErrEmptyStack}, nil
+	case "EQUALVERIFY":
+		return []ErrorCode{ErrEqualVerify}, nil
+	case "NULLFAIL":
+		return []ErrorCode{ErrSigNullDummy}, nil
+	case "SIG_HIGH_S":
+		return []ErrorCode{ErrSigHighS}, nil
+	case "SIG_HASHTYPE":
+		return []ErrorCode{ErrInvalidSigHashType}, nil
+	case "SIG_NULLDUMMY":
+		return []ErrorCode{ErrSigNullDummy}, nil
+	case "SIG_PUSHONLY":
+		return []ErrorCode{ErrNotPushOnly}, nil
+	case "CLEANSTACK":
+		return []ErrorCode{ErrCleanStack}, nil
+	case "BAD_OPCODE":
+		return []ErrorCode{ErrReservedOpcode, ErrMalformedPush}, nil
+	case "UNBALANCED_CONDITIONAL":
+		return []ErrorCode{ErrUnbalancedConditional,
+			ErrInvalidStackOperation}, nil
+	case "OP_RETURN":
+		return []ErrorCode{ErrEarlyReturn}, nil
+	case "VERIFY":
+		return []ErrorCode{ErrVerify}, nil
+	case "INVALID_STACK_OPERATION", "INVALID_ALTSTACK_OPERATION":
+		return []ErrorCode{ErrInvalidStackOperation}, nil
+	case "DISABLED_OPCODE":
+		return []ErrorCode{ErrDisabledOpcode}, nil
+	case "DISCOURAGE_UPGRADABLE_NOPS":
+		return []ErrorCode{ErrDiscourageUpgradableNOPs}, nil
+	case "PUSH_SIZE":
+		return []ErrorCode{ErrElementTooBig}, nil
+	case "OP_COUNT":
+		return []ErrorCode{ErrTooManyOperations}, nil
+	case "STACK_SIZE":
+		return []ErrorCode{ErrStackOverflow}, nil
+	case "SCRIPT_SIZE":
+		return []ErrorCode{ErrScriptTooBig}, nil
+	case "PUBKEY_COUNT":
+		return []ErrorCode{ErrInvalidPubKeyCount}, nil
+	case "SIG_COUNT":
+		return []ErrorCode{ErrInvalidSignatureCount}, nil
+	case "MINIMALDATA":
+		return []ErrorCode{ErrMinimalData}, nil
+	case "NEGATIVE_LOCKTIME":
+		return []ErrorCode{ErrNegativeLockTime}, nil
+	case "UNSATISFIED_LOCKTIME":
+		return []ErrorCode{ErrUnsatisfiedLockTime}, nil
+	}
+
+	return nil, fmt.Errorf("unrecognized expected result in test data: %v",
+		expected)
+}
+
 // createSpendTx generates a basic spending transaction given the passed
 // signature and public key scripts.
 func createSpendingTx(sigScript, pkScript []byte) *wire.MsgTx {
@@ -184,139 +264,155 @@ func createSpendingTx(sigScript, pkScript []byte) *wire.MsgTx {
 	return spendingTx
 }
 
-// TestScriptInvalidTests ensures all of the tests in script_invalid.json fail
+// testScripts ensures all of the passed script tests execute with the expected
-// as expected.
+// results with or without using a signature cache, as specified by the
-func TestScriptInvalidTests(t *testing.T) {
+// parameter.
-	file, err := ioutil.ReadFile("data/script_invalid.json")
+func testScripts(t *testing.T, tests [][]interface{}, useSigCache bool) {
-	if err != nil {
+	// Create a signature cache to use only if requested.
-		t.Errorf("TestBitcoindInvalidTests: %v\n", err)
+	var sigCache *SigCache
-		return
+	if useSigCache {
+		sigCache = NewSigCache(10)
 	}
 
-	var tests [][]string
+	for i, test := range tests {
-	err = json.Unmarshal(file, &tests)
+		// "Format is: [[wit..., amount]?, scriptSig, scriptPubKey,
-	if err != nil {
+		//    flags, expected_scripterror, ... comments]"
-		t.Errorf("TestBitcoindInvalidTests couldn't Unmarshal: %v",
-			err)
-		return
-	}
-	sigCache := NewSigCache(10)
 
-	sigCacheToggle := []bool{true, false}
+		// Skip single line comments.
-	for _, useSigCache := range sigCacheToggle {
+		if len(test) == 1 {
-		for i, test := range tests {
+			continue
-			// Skip comments
+		}
-			if len(test) == 1 {
-				continue
-			}
-			name, err := testName(test)
-			if err != nil {
-				t.Errorf("TestBitcoindInvalidTests: invalid test #%d",
-					i)
-				continue
-			}
-			scriptSig, err := parseShortForm(test[0])
-			if err != nil {
-				t.Errorf("%s: can't parse scriptSig; %v", name, err)
-				continue
-			}
-			scriptPubKey, err := parseShortForm(test[1])
-			if err != nil {
-				t.Errorf("%s: can't parse scriptPubkey; %v", name, err)
-				continue
-			}
-			flags, err := parseScriptFlags(test[2])
-			if err != nil {
-				t.Errorf("%s: %v", name, err)
-				continue
-			}
-			tx := createSpendingTx(scriptSig, scriptPubKey)
 
-			var vm *Engine
+		// Construct a name for the test based on the comment and test
-			if useSigCache {
+		// data.
-				vm, err = NewEngine(scriptPubKey, tx, 0, flags, sigCache)
+		name, err := scriptTestName(test)
-			} else {
+		if err != nil {
-				vm, err = NewEngine(scriptPubKey, tx, 0, flags, nil)
+			t.Errorf("TestScripts: invalid test #%d: %v", i, err)
-			}
+			continue
+		}
 
-			if err == nil {
+		// When the first field of the test data is a slice it contains
-				if err := vm.Execute(); err == nil {
+		// witness data and everything else is offset by 1 as a result.
-					t.Errorf("%s test succeeded when it "+
+		witnessOffset := 0
-						"should have failed\n", name)
+		witnessData, ok := test[0].([]interface{})
-				}
+		if ok {
+			witnessOffset++
+
+		}
+		_ = witnessData // Unused for now until segwit code lands
+
+		// Extract and parse the signature script from the test fields.
+		scriptSigStr, ok := test[witnessOffset].(string)
+		if !ok {
+			t.Errorf("%s: signature script is not a string", name)
+			continue
+		}
+		scriptSig, err := parseShortForm(scriptSigStr)
+		if err != nil {
+			t.Errorf("%s: can't parse signature script: %v", name,
+				err)
+			continue
+		}
+
+		// Extract and parse the public key script from the test fields.
+		scriptPubKeyStr, ok := test[witnessOffset+1].(string)
+		if !ok {
+			t.Errorf("%s: public key script is not a string", name)
+			continue
+		}
+		scriptPubKey, err := parseShortForm(scriptPubKeyStr)
+		if err != nil {
+			t.Errorf("%s: can't parse public key script: %v", name,
+				err)
+			continue
+		}
+
+		// Extract and parse the script flags from the test fields.
+		flagsStr, ok := test[witnessOffset+2].(string)
+		if !ok {
+			t.Errorf("%s: flags field is not a string", name)
+			continue
+		}
+		flags, err := parseScriptFlags(flagsStr)
+		if err != nil {
+			t.Errorf("%s: %v", name, err)
+			continue
+		}
+
+		// Extract and parse the expected result from the test fields.
+		//
+		// Convert the expected result string into the allowed script
+		// error codes.  This is necessary because txscript is more
+		// fine grained with its errors than the reference test data, so
+		// some of the reference test data errors map to more than one
+		// possibility.
+		resultStr, ok := test[witnessOffset+3].(string)
+		if !ok {
+			t.Errorf("%s: result field is not a string", name)
+			continue
+		}
+		allowedErrorCodes, err := parseExpectedResult(resultStr)
+		if err != nil {
+			t.Errorf("%s: %v", name, err)
+			continue
+		}
+
+		// Generate a transaction pair such that one spends from the
+		// other and the provided signature and public key scripts are
+		// used, then create a new engine to execute the scripts.
+		tx := createSpendingTx(scriptSig, scriptPubKey)
+		vm, err := NewEngine(scriptPubKey, tx, 0, flags, sigCache)
+		if err == nil {
+			err = vm.Execute()
+		}
+
+		// Ensure there were no errors when the expected result is OK.
+		if resultStr == "OK" {
+			if err != nil {
+				t.Errorf("%s failed to execute: %v", name, err)
+			}
+			continue
+		}
+
+		// At this point an error was expected so ensure the result of
+		// the execution matches it.
+		success := false
+		for _, code := range allowedErrorCodes {
+			if IsErrorCode(err, code) {
+				success = true
+				break
+			}
+		}
+		if !success {
+			if serr, ok := err.(Error); ok {
+				t.Errorf("%s: want error codes %v, got %v", name,
+					allowedErrorCodes, serr.ErrorCode)
 				continue
 			}
+			t.Errorf("%s: want error codes %v, got err: %v (%T)",
+				name, allowedErrorCodes, err, err)
+			continue
 		}
 	}
 }
 
-// TestScriptValidTests ensures all of the tests in script_valid.json pass as
+// TestScripts ensures all of the tests in script_tests.json execute with the
-// expected.
+// expected results as defined in the test data.
-func TestScriptValidTests(t *testing.T) {
+func TestScripts(t *testing.T) {
-	file, err := ioutil.ReadFile("data/script_valid.json")
+	file, err := ioutil.ReadFile("data/script_tests.json")
 	if err != nil {
-		t.Errorf("TestBitcoinValidTests: %v\n", err)
+		t.Fatalf("TestScripts: %v\n", err)
-		return
 	}
 
-	var tests [][]string
+	var tests [][]interface{}
 	err = json.Unmarshal(file, &tests)
 	if err != nil {
-		t.Errorf("TestBitcoindValidTests couldn't Unmarshal: %v",
+		t.Fatalf("TestScripts couldn't Unmarshal: %v", err)
-			err)
-		return
 	}
 
-	sigCache := NewSigCache(10)
+	// Run all script tests with and without the signature cache.
-
+	testScripts(t, tests, true)
-	sigCacheToggle := []bool{true, false}
+	testScripts(t, tests, false)
-	for _, useSigCache := range sigCacheToggle {
-		for i, test := range tests {
-			// Skip comments
-			if len(test) == 1 {
-				continue
-			}
-			name, err := testName(test)
-			if err != nil {
-				t.Errorf("TestBitcoindValidTests: invalid test #%d",
-					i)
-				continue
-			}
-			scriptSig, err := parseShortForm(test[0])
-			if err != nil {
-				t.Errorf("%s: can't parse scriptSig; %v", name, err)
-				continue
-			}
-			scriptPubKey, err := parseShortForm(test[1])
-			if err != nil {
-				t.Errorf("%s: can't parse scriptPubkey; %v", name, err)
-				continue
-			}
-			flags, err := parseScriptFlags(test[2])
-			if err != nil {
-				t.Errorf("%s: %v", name, err)
-				continue
-			}
-			tx := createSpendingTx(scriptSig, scriptPubKey)
-
-			var vm *Engine
-			if useSigCache {
-				vm, err = NewEngine(scriptPubKey, tx, 0, flags, sigCache)
-			} else {
-				vm, err = NewEngine(scriptPubKey, tx, 0, flags, nil)
-			}
-
-			if err != nil {
-				t.Errorf("%s failed to create script: %v", name, err)
-				continue
-			}
-			err = vm.Execute()
-			if err != nil {
-				t.Errorf("%s failed to execute: %v", name, err)
-				continue
-			}
-		}
-	}
 }
 
 // testVecF64ToUint32 properly handles conversion of float64s read from the JSON
@@ -336,15 +432,13 @@ func testVecF64ToUint32(f float64) uint32 {
 func TestTxInvalidTests(t *testing.T) {
 	file, err := ioutil.ReadFile("data/tx_invalid.json")
 	if err != nil {
-		t.Errorf("TestTxInvalidTests: %v\n", err)
+		t.Fatalf("TestTxInvalidTests: %v\n", err)
-		return
 	}
 
 	var tests [][]interface{}
 	err = json.Unmarshal(file, &tests)
 	if err != nil {
-		t.Errorf("TestTxInvalidTests couldn't Unmarshal: %v\n", err)
+		t.Fatalf("TestTxInvalidTests couldn't Unmarshal: %v\n", err)
-		return
 	}
 
 	// form is either:
@@ -479,15 +573,13 @@ testloop:
 func TestTxValidTests(t *testing.T) {
 	file, err := ioutil.ReadFile("data/tx_valid.json")
 	if err != nil {
-		t.Errorf("TestTxValidTests: %v\n", err)
+		t.Fatalf("TestTxValidTests: %v\n", err)
-		return
 	}
 
 	var tests [][]interface{}
 	err = json.Unmarshal(file, &tests)
 	if err != nil {
-		t.Errorf("TestTxValidTests couldn't Unmarshal: %v\n", err)
+		t.Fatalf("TestTxValidTests couldn't Unmarshal: %v\n", err)
-		return
 	}
 
 	// form is either:
@@ -621,16 +713,14 @@ testloop:
 func TestCalcSignatureHash(t *testing.T) {
 	file, err := ioutil.ReadFile("data/sighash.json")
 	if err != nil {
-		t.Errorf("TestCalcSignatureHash: %v\n", err)
+		t.Fatalf("TestCalcSignatureHash: %v\n", err)
-		return
 	}
 
 	var tests [][]interface{}
 	err = json.Unmarshal(file, &tests)
 	if err != nil {
-		t.Errorf("TestCalcSignatureHash couldn't Unmarshal: %v\n",
+		t.Fatalf("TestCalcSignatureHash couldn't Unmarshal: %v\n",
 			err)
-		return
 	}
 
 	for i, test := range tests {