Rework the pre-computed table generation and load.

This commit reworks the way that the pre-computed table which is used to
accelerate scalar base multiple is generated and loaded to make use of the
go generate infrastructure and greatly reduce the memory needed to compile
as well as speed up the compile.

Previously, the table was being generated using the in-memory
representation directly written into the file.  Since the table has a very
large number of entries, the Go compiler was taking up to nearly 1GB to
compile.  It also took a comparatively long period of time to compile.

Instead, this commit modifies the generated table to be a serialized,
compressed, and base64-encoded byte slice.  At init time, this process is
reversed to create the in-memory representation.  This approach provides
fast compile times with much lower memory needed to compile (16MB versus
1GB).  In addition, the init time cost is extremely low, especially as
compared to computing the entire table.

Finally, the automatic generation wasn't really automatic.  It is now
fully automatic with 'go generate'.

btcec.go

@@ -8,7 +8,7 @@ package btcec
 
 // References:
 //   [SECG]: Recommended Elliptic Curve Domain Parameters
-//     http://www.secg.org/download/aid-784/sec2-v2.pdf
+//     http://www.secg.org/sec2-v2.pdf
 
 // This package operates, internally, on Jacobian coordinates. For a given
 // (x, y) position on the curve, the Jacobian coordinates are (x1, y1, z1)
@@ -658,7 +658,6 @@ func (curve *KoblitzCurve) QPlus1Div4() *big.Int {
 	return curve.q
 }
 
-// Curve parameters taken from: http://www.secg.org/sec2-v2.pdf
 var initonce sync.Once
 var secp256k1 KoblitzCurve
 
@@ -667,7 +666,7 @@ func initAll() {
 }
 
 func initS256() {
-	// See [SECG] section 2.7.1
+	// Curve parameters taken from [SECG] section 2.4.1.
 	secp256k1.CurveParams = new(elliptic.CurveParams)
 	secp256k1.P, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEFFFFFC2F", 16)
 	secp256k1.N, _ = new(big.Int).SetString("FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFEBAAEDCE6AF48A03BBFD25E8CD0364141", 16)
@@ -678,7 +677,13 @@ func initS256() {
 	secp256k1.H = 1
 	secp256k1.q = new(big.Int).Div(new(big.Int).Add(secp256k1.P,
 		big.NewInt(1)), big.NewInt(4))
-	secp256k1.bytePoints = &secp256k1BytePoints
+
+	// Deserialize and set the pre-computed table used to accelerate scalar
+	// base multiplication.  This is hard-coded data, so any errors are
+	// panics because it means something is wrong in the source code.
+	if err := loadS256BytePoints(); err != nil {
+		panic(err)
+	}
 }
 
 // S256 returns a Curve which implements secp256k1.

genprecomps.go

@@ -0,0 +1,54 @@
+// Copyright 2015 Conformal Systems LLC. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// This file is ignored during the regular build due to the following build tag.
+// It is called by go generate and used to automatically generate pre-computed
+// tables used to accelerate operations.
+// +build ignore
+
+package main
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/base64"
+	"fmt"
+	"log"
+	"os"
+
+	"github.com/btcsuite/btcec"
+)
+
+func main() {
+	fi, err := os.Create("secp256k1.go")
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer fi.Close()
+
+	// Compress the serialized byte points.
+	serialized := btcec.S256().SerializedBytePoints()
+	var compressed bytes.Buffer
+	w := zlib.NewWriter(&compressed)
+	if _, err := w.Write(serialized); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+	w.Close()
+
+	// Encode the compressed byte points with base64.
+	encoded := make([]byte, base64.StdEncoding.EncodedLen(compressed.Len()))
+	base64.StdEncoding.Encode(encoded, compressed.Bytes())
+
+	fmt.Fprintln(fi, "// Copyright (c) 2015 Conformal Systems LLC.")
+	fmt.Fprintln(fi, "// Use of this source code is governed by an ISC")
+	fmt.Fprintln(fi, "// license that can be found in the LICENSE file.")
+	fmt.Fprintln(fi)
+	fmt.Fprintln(fi, "package btcec")
+	fmt.Fprintln(fi)
+	fmt.Fprintln(fi, "// Auto-generated file (see genprecomps.go)")
+	fmt.Fprintln(fi, "// DO NOT EDIT")
+	fmt.Fprintln(fi)
+	fmt.Fprintf(fi, "var secp256k1BytePoints = []byte(%q)\n", encoded)
+}

gensecp256k1.go

@@ -1,13 +1,18 @@
-// Copyright (c) 2014 Conformal Systems LLC.
+// Copyright (c) 2014-2015 Conformal Systems LLC.
 // Use of this source code is governed by an ISC
 // license that can be found in the LICENSE file.
+
+// This file is ignored during the regular build due to the following build tag.
+// This build tag is set during go generate.
 // +build gensecp256k1
 
 package btcec
 
-import (
-	"fmt"
-)
+import "encoding/binary"
+
+// secp256k1BytePoints are dummy points used so the code which generates the
+// real values can compile.
+var secp256k1BytePoints = []byte{}
 
 // getDoublingPoints returns all the possible G^(2^i) for i in
 // 0..n-1 where n is the curve's bit size (256 in the case of secp256k1)
@@ -27,27 +32,23 @@ func (curve *KoblitzCurve) getDoublingPoints() [][3]fieldVal {
 	return doublingPoints
 }
 
-// PrintBytePoints prints all the possible points per 8-bit window.
-// normally, this is used to generate secp256k1.go
-func (curve *KoblitzCurve) PrintBytePoints() {
+// SerializedBytePoints returns a serialized byte slice which contains all of
+// the possible points per 8-bit window.  This is used to when generating
+// secp256k1.go.
+func (curve *KoblitzCurve) SerializedBytePoints() []byte {
 	bitSize := curve.Params().BitSize
 	byteSize := bitSize / 8
 	doublingPoints := curve.getDoublingPoints()
-	fmt.Println("// Copyright (c) 2014 Conformal Systems LLC.")
-	fmt.Println("// Use of this source code is governed by an ISC")
-	fmt.Println("// license that can be found in the LICENSE file.")
-	fmt.Println()
-	fmt.Println("package btcec")
-	fmt.Println()
-	fmt.Println("// Auto-generated file (see gensecp256k1.go)")
-	fmt.Printf("var secp256k1BytePoints = [%d][256][3]fieldVal{\n", byteSize)
+
 	// Segregate the bits into byte-sized windows
+	serialized := make([]byte, byteSize*256*3*10*4)
+	offset := 0
 	for byteNum := 0; byteNum < byteSize; byteNum++ {
-		fmt.Printf("\t{\n")
-		// grab the 8 bits that make up this byte from doublingPoints
+		// Grab the 8 bits that make up this byte from doublingPoints.
 		startingBit := 8 * (byteSize - byteNum - 1)
 		computingPoints := doublingPoints[startingBit : startingBit+8]
-		// compute all points in this window
+
+		// Compute all points in this window and serialize them.
 		for i := 0; i < 256; i++ {
 			px, py, pz := new(fieldVal), new(fieldVal), new(fieldVal)
 			for j := 0; j < 8; j++ {
@@ -56,11 +57,20 @@ func (curve *KoblitzCurve) PrintBytePoints() {
 						&computingPoints[j][1], &computingPoints[j][2], px, py, pz)
 				}
 			}
-			fmt.Printf("\t\t{\n\t\t\tfieldVal{[10]uint32{%d, %d, %d, %d, %d, %d, %d, %d, %d, %d}},\n", px.n[0], px.n[1], px.n[2], px.n[3], px.n[4], px.n[5], px.n[6], px.n[7], px.n[8], px.n[9])
-			fmt.Printf("\t\t\tfieldVal{[10]uint32{%d, %d, %d, %d, %d, %d, %d, %d, %d, %d}},\n", py.n[0], py.n[1], py.n[2], py.n[3], py.n[4], py.n[5], py.n[6], py.n[7], py.n[8], py.n[9])
-			fmt.Printf("\t\t\tfieldVal{[10]uint32{%d, %d, %d, %d, %d, %d, %d, %d, %d, %d}},\n\t\t},\n", pz.n[0], pz.n[1], pz.n[2], pz.n[3], pz.n[4], pz.n[5], pz.n[6], pz.n[7], pz.n[8], pz.n[9])
+			for i := 0; i < 10; i++ {
+				binary.LittleEndian.PutUint32(serialized[offset:], px.n[i])
+				offset += 4
+			}
+			for i := 0; i < 10; i++ {
+				binary.LittleEndian.PutUint32(serialized[offset:], py.n[i])
+				offset += 4
+			}
+			for i := 0; i < 10; i++ {
+				binary.LittleEndian.PutUint32(serialized[offset:], pz.n[i])
+				offset += 4
+			}
 		}
-		fmt.Printf("\t},\n")
 	}
-	fmt.Printf("}\n")
+
+	return serialized
 }

precompute.go

@@ -0,0 +1,71 @@
+// Copyright 2015 Conformal Systems LLC. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package btcec
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/base64"
+	"encoding/binary"
+	"io/ioutil"
+)
+
+//go:generate go run -tags gensecp256k1 genprecomps.go
+
+// loadS256BytePoints decompresses and deserializes the pre-computed byte points
+// used to accelerate scalar base multiplication for the secp256k1 curve.  This
+// approach is used since it allows the compile to use significantly less ram
+// and be performed much faster than it is with hard-coding the final in-memory
+// data structure.  At the same time, it is quite fast to generate the in-memory
+// data structure at init time with this approach versus computing the table.
+func loadS256BytePoints() error {
+	// There will be no byte points to load when generating them.
+	bp := secp256k1BytePoints
+	if len(secp256k1BytePoints) == 0 {
+		return nil
+	}
+
+	// Decompress the pre-computed table used to accelerate scalar base
+	// multiplication.
+	decoded := make([]byte, base64.StdEncoding.DecodedLen(len(bp)))
+	if _, err := base64.StdEncoding.Decode(decoded, bp); err != nil {
+		return err
+	}
+	r, err := zlib.NewReader(bytes.NewReader(decoded))
+	if err != nil {
+		return err
+	}
+	serialized, err := ioutil.ReadAll(r)
+	if err != nil {
+		return err
+	}
+
+	// Deserialize the precomputed byte points and set the curve to them.
+	offset := 0
+	var bytePoints [32][256][3]fieldVal
+	for byteNum := 0; byteNum < 32; byteNum++ {
+		// All points in this window.
+		for i := 0; i < 256; i++ {
+			px, py, pz := new(fieldVal), new(fieldVal), new(fieldVal)
+			for i := 0; i < 10; i++ {
+				px.n[i] = binary.LittleEndian.Uint32(serialized[offset:])
+				offset += 4
+			}
+			for i := 0; i < 10; i++ {
+				py.n[i] = binary.LittleEndian.Uint32(serialized[offset:])
+				offset += 4
+			}
+			for i := 0; i < 10; i++ {
+				pz.n[i] = binary.LittleEndian.Uint32(serialized[offset:])
+				offset += 4
+			}
+			bytePoints[byteNum][i][0] = *px
+			bytePoints[byteNum][i][1] = *py
+			bytePoints[byteNum][i][2] = *pz
+		}
+	}
+	secp256k1.bytePoints = &bytePoints
+	return nil
+}