Add ordered maps package

extras/orderedmap/ordered_map.go

@@ -0,0 +1,301 @@
+package orderedmap
+
+// mostly from https://github.com/iancoleman/orderedmap
+
+import (
+	"encoding/json"
+	"fmt"
+	"sort"
+	"strings"
+	"sync"
+
+	"github.com/lbryio/lbry.go/extras/errors"
+)
+
+type keyIndex struct {
+	Key   string
+	Index int
+}
+
+type byIndex []keyIndex
+
+func (a byIndex) Len() int           { return len(a) }
+func (a byIndex) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a byIndex) Less(i, j int) bool { return a[i].Index < a[j].Index }
+
+type Map struct {
+	l      sync.RWMutex
+	keys   []string
+	values map[string]interface{}
+}
+
+func New() *Map {
+	o := Map{}
+	o.l = sync.RWMutex{}
+	o.keys = []string{}
+	o.values = map[string]interface{}{}
+	return &o
+}
+
+func (o *Map) Get(key string) (interface{}, bool) {
+	o.l.RLock()
+	defer o.l.RUnlock()
+	val, exists := o.values[key]
+	return val, exists
+}
+
+func (o *Map) Set(key string, value interface{}) {
+	o.l.Lock()
+	defer o.l.Unlock()
+	_, exists := o.values[key]
+	if !exists {
+		o.keys = append(o.keys, key)
+	}
+	o.values[key] = value
+}
+
+const outOfRange = "position value %d is outside of the range %d - %d"
+
+//InsertAt This is a zero based position index: 0,1,2,3..n(from left) OR -1,-2,-3...-n(from right) where -1 is the last place.
+func (o *Map) InsertAt(key string, value interface{}, position int) error {
+	o.l.Lock()
+	defer o.l.Unlock()
+	var index = position
+	if position < 0 {
+		// support indexing from the back: -1 = last in array.
+		index += len(o.keys) + 1
+		if index < 0 || index > len(o.keys) {
+			return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1))
+		}
+	} else if index > len(o.keys) {
+		return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1))
+	}
+	_, exists := o.values[key]
+	if !exists {
+		// left + key + right
+		o.keys = append(o.keys[0:index], append([]string{key}, o.keys[index:]...)...)
+	}
+	o.values[key] = value
+
+	return nil
+}
+
+func (o *Map) Prepend(key string, value interface{}) {
+	o.l.Lock()
+	defer o.l.Unlock()
+	_, exists := o.values[key]
+	if !exists {
+		o.keys = append([]string{key}, o.keys...)
+	}
+	o.values[key] = value
+}
+
+func (o *Map) Delete(key string) {
+	o.l.Lock()
+	defer o.l.Unlock()
+	// check key is in use
+	_, ok := o.values[key]
+	if !ok {
+		return
+	}
+	// remove from keys
+	for i, k := range o.keys {
+		if k == key {
+			o.keys = append(o.keys[:i], o.keys[i+1:]...)
+			break
+		}
+	}
+	// remove from values
+	delete(o.values, key)
+}
+
+func (o *Map) Keys() []string {
+	o.l.RLock()
+	defer o.l.RUnlock()
+	return o.keys
+}
+
+func (o *Map) UnmarshalJSON(b []byte) error {
+	o.l.Lock()
+	defer o.l.Unlock()
+	m := map[string]interface{}{}
+	if err := json.Unmarshal(b, &m); err != nil {
+		return errors.Err(err)
+	}
+	s := string(b)
+	mapToOrderedMap(o, s, m)
+	return nil
+}
+
+func mapToOrderedMap(o *Map, s string, m map[string]interface{}) {
+	// Get the order of the keys
+	orderedKeys := []keyIndex{}
+	for k := range m {
+		kEscaped := strings.Replace(k, `"`, `\"`, -1)
+		kQuoted := `"` + kEscaped + `"`
+		// Find how much content exists before this key.
+		// If all content from this key and after is replaced with a close
+		// brace, it should still form a valid json string.
+		sTrimmed := s
+		for len(sTrimmed) > 0 {
+			lastIndex := strings.LastIndex(sTrimmed, kQuoted)
+			if lastIndex == -1 {
+				break
+			}
+			sTrimmed = sTrimmed[0:lastIndex]
+			sTrimmed = strings.TrimSpace(sTrimmed)
+			if len(sTrimmed) > 0 && sTrimmed[len(sTrimmed)-1] == ',' {
+				sTrimmed = sTrimmed[0 : len(sTrimmed)-1]
+			}
+			maybeValidJson := sTrimmed + "}"
+			testMap := map[string]interface{}{}
+			err := json.Unmarshal([]byte(maybeValidJson), &testMap)
+			if err == nil {
+				// record the position of this key in s
+				ki := keyIndex{
+					Key:   k,
+					Index: len(sTrimmed),
+				}
+				orderedKeys = append(orderedKeys, ki)
+				// shorten the string to get the next key
+				startOfValueIndex := lastIndex + len(kQuoted)
+				valueStr := s[startOfValueIndex : len(s)-1]
+				valueStr = strings.TrimSpace(valueStr)
+				if len(valueStr) > 0 && valueStr[0] == ':' {
+					valueStr = valueStr[1:]
+				}
+				valueStr = strings.TrimSpace(valueStr)
+				if valueStr[0] == '{' {
+					// if the value for this key is a map, convert it to an orderedmap.
+					// find end of valueStr by removing everything after last }
+					// until it forms valid json
+					hasValidJson := false
+					i := 1
+					for i < len(valueStr) && !hasValidJson {
+						if valueStr[i] != '}' {
+							i = i + 1
+							continue
+						}
+						subTestMap := map[string]interface{}{}
+						testValue := valueStr[0 : i+1]
+						err = json.Unmarshal([]byte(testValue), &subTestMap)
+						if err == nil {
+							hasValidJson = true
+							valueStr = testValue
+							break
+						}
+						i = i + 1
+					}
+					// convert to orderedmap
+					if hasValidJson {
+						mkTyped := m[k].(map[string]interface{})
+						oo := &Map{}
+						mapToOrderedMap(oo, valueStr, mkTyped)
+						m[k] = oo
+					}
+				} else if valueStr[0] == '[' {
+					// if the value for this key is a []interface, convert any map items to an orderedmap.
+					// find end of valueStr by removing everything after last ]
+					// until it forms valid json
+					hasValidJson := false
+					i := 1
+					for i < len(valueStr) && !hasValidJson {
+						if valueStr[i] != ']' {
+							i = i + 1
+							continue
+						}
+						subTestSlice := []interface{}{}
+						testValue := valueStr[0 : i+1]
+						err = json.Unmarshal([]byte(testValue), &subTestSlice)
+						if err == nil {
+							hasValidJson = true
+							valueStr = testValue
+							break
+						}
+						i = i + 1
+					}
+					if hasValidJson {
+						itemsStr := valueStr[1 : len(valueStr)-1]
+						// get next item in the slice
+						itemIndex := 0
+						startItem := 0
+						endItem := 0
+						for endItem < len(itemsStr) {
+							if itemsStr[endItem] != ',' && endItem < len(itemsStr)-1 {
+								endItem = endItem + 1
+								continue
+							}
+							// if this substring compiles to json, it's the next item
+							possibleItemStr := strings.TrimSpace(itemsStr[startItem:endItem])
+							var possibleItem interface{}
+							err = json.Unmarshal([]byte(possibleItemStr), &possibleItem)
+							if err != nil {
+								endItem = endItem + 1
+								continue
+							}
+							// if item is map, convert to orderedmap
+							if possibleItemStr[0] == '{' {
+								mkTyped := m[k].([]interface{})
+								mkiTyped := mkTyped[itemIndex].(map[string]interface{})
+								oo := &Map{}
+								mapToOrderedMap(oo, possibleItemStr, mkiTyped)
+								// replace original map with orderedmap
+								mkTyped[itemIndex] = oo
+								m[k] = mkTyped
+							}
+							// remove this item from itemsStr
+							startItem = endItem + 1
+							endItem = endItem + 1
+							itemIndex = itemIndex + 1
+						}
+					}
+				}
+				break
+			}
+		}
+	}
+	// Sort the keys
+	sort.Sort(byIndex(orderedKeys))
+	// Convert sorted keys to string slice
+	k := []string{}
+	for _, ki := range orderedKeys {
+		k = append(k, ki.Key)
+	}
+	// Set the Map values
+	o.values = m
+	o.keys = k
+}
+
+func (o *Map) Copy() *Map {
+	new := New()
+
+	for _, k := range o.keys {
+		v, _ := o.Get(k)
+		new.Set(k, v)
+	}
+
+	return new
+}
+
+func (o *Map) MarshalJSON() ([]byte, error) {
+	o.l.RLock()
+	defer o.l.RUnlock()
+	s := "{"
+	for _, k := range o.keys {
+		// add key
+		kEscaped := strings.Replace(k, `"`, `\"`, -1)
+		s = s + `"` + kEscaped + `":`
+		// add value
+		v := o.values[k]
+		vBytes, err := json.Marshal(v)
+		if err != nil {
+			return []byte{}, errors.Err(err)
+		}
+		s = s + string(vBytes) + ","
+	}
+	if len(o.keys) > 0 {
+		s = s[0 : len(s)-1]
+	}
+	s = s + "}"
+	return []byte(s), nil
+}

extras/orderedmap/ordered_map_test.go

@@ -0,0 +1,478 @@
+package orderedmap
+
+import (
+	"encoding/json"
+	"fmt"
+	"math/rand"
+	"strconv"
+	"sync"
+	"testing"
+	"time"
+
+	"github.com/spf13/cast"
+)
+
+func TestOrderedMap(t *testing.T) {
+	o := New()
+	// number
+	o.Set("number", 3)
+	v, _ := o.Get("number")
+	if v.(int) != 3 {
+		t.Error("Set number")
+	}
+	// string
+	o.Set("string", "x")
+	v, _ = o.Get("string")
+	if v.(string) != "x" {
+		t.Error("Set string")
+	}
+	// string slice
+	o.Set("strings", []string{
+		"t",
+		"u",
+	})
+	v, _ = o.Get("strings")
+	if v.([]string)[0] != "t" {
+		t.Error("Set strings first index")
+	}
+	if v.([]string)[1] != "u" {
+		t.Error("Set strings second index")
+	}
+	// mixed slice
+	o.Set("mixed", []interface{}{
+		1,
+		"1",
+	})
+	v, _ = o.Get("mixed")
+	if v.([]interface{})[0].(int) != 1 {
+		t.Error("Set mixed int")
+	}
+	if v.([]interface{})[1].(string) != "1" {
+		t.Error("Set mixed string")
+	}
+	// overriding existing key
+	o.Set("number", 4)
+	v, _ = o.Get("number")
+	if v.(int) != 4 {
+		t.Error("Override existing key")
+	}
+	// Keys method
+	keys := o.Keys()
+	expectedKeys := []string{
+		"number",
+		"string",
+		"strings",
+		"mixed",
+	}
+	for i := range keys {
+		if keys[i] != expectedKeys[i] {
+			t.Error("Keys method", keys[i], "!=", expectedKeys[i])
+		}
+	}
+	for i := range expectedKeys {
+		if keys[i] != expectedKeys[i] {
+			t.Error("Keys method", keys[i], "!=", expectedKeys[i])
+		}
+	}
+	// delete
+	o.Delete("strings")
+	o.Delete("not a key being used")
+	if len(o.Keys()) != 3 {
+		t.Error("Delete method")
+	}
+	_, ok := o.Get("strings")
+	if ok {
+		t.Error("Delete did not remove 'strings' key")
+	}
+}
+
+func TestBlankMarshalJSON(t *testing.T) {
+	o := New()
+	// blank map
+	b, err := json.Marshal(o)
+	if err != nil {
+		t.Error("Marshalling blank map to json", err)
+	}
+	s := string(b)
+	// check json is correctly ordered
+	if s != `{}` {
+		t.Error("JSON Marshaling blank map value is incorrect", s)
+	}
+	// convert to indented json
+	bi, err := json.MarshalIndent(o, "", "  ")
+	if err != nil {
+		t.Error("Marshalling indented json for blank map", err)
+	}
+	si := string(bi)
+	ei := `{}`
+	if si != ei {
+		fmt.Println(ei)
+		fmt.Println(si)
+		t.Error("JSON MarshalIndent blank map value is incorrect", si)
+	}
+}
+
+func TestMarshalJSON(t *testing.T) {
+	o := New()
+	// number
+	o.Set("number", 3)
+	// string
+	o.Set("string", "x")
+	// new value keeps key in old position
+	o.Set("number", 4)
+	// keys not sorted alphabetically
+	o.Set("z", 1)
+	o.Set("a", 2)
+	o.Set("b", 3)
+	// slice
+	o.Set("slice", []interface{}{
+		"1",
+		1,
+	})
+	// orderedmap
+	v := New()
+	v.Set("e", 1)
+	v.Set("a", 2)
+	o.Set("orderedmap", v)
+	// double quote in key
+	o.Set(`test"ing`, 9)
+	// convert to json
+	b, err := json.Marshal(o)
+	if err != nil {
+		t.Error("Marshalling json", err)
+	}
+	s := string(b)
+	// check json is correctly ordered
+	if s != `{"number":4,"string":"x","z":1,"a":2,"b":3,"slice":["1",1],"orderedmap":{"e":1,"a":2},"test\"ing":9}` {
+		t.Error("JSON Marshal value is incorrect", s)
+	}
+	// convert to indented json
+	bi, err := json.MarshalIndent(o, "", "  ")
+	if err != nil {
+		t.Error("Marshalling indented json", err)
+	}
+	si := string(bi)
+	ei := `{
+  "number": 4,
+  "string": "x",
+  "z": 1,
+  "a": 2,
+  "b": 3,
+  "slice": [
+    "1",
+    1
+  ],
+  "orderedmap": {
+    "e": 1,
+    "a": 2
+  },
+  "test\"ing": 9
+}`
+	if si != ei {
+		fmt.Println(ei)
+		fmt.Println(si)
+		t.Error("JSON MarshalIndent value is incorrect", si)
+	}
+}
+
+func TestUnmarshalJSON(t *testing.T) {
+	s := `{
+  "number": 4,
+  "string": "x",
+  "z": 1,
+  "a": "should not break with unclosed { character in value",
+  "b": 3,
+  "slice": [
+    "1",
+    1
+  ],
+  "orderedmap": {
+    "e": 1,
+    "a { nested key with brace": "with a }}}} }} {{{ brace value",
+	"after": {
+		"link": "test {{{ with even deeper nested braces }"
+	}
+  },
+  "test\"ing": 9,
+  "after": 1,
+  "multitype_array": [
+    "test",
+	1,
+	{ "map": "obj", "it" : 5, ":colon in key": "colon: in value" }
+  ],
+  "should not break with { character in key": 1
+}`
+	o := New()
+	err := json.Unmarshal([]byte(s), &o)
+	if err != nil {
+		t.Error("JSON Unmarshal error", err)
+	}
+	// Check the root keys
+	expectedKeys := []string{
+		"number",
+		"string",
+		"z",
+		"a",
+		"b",
+		"slice",
+		"orderedmap",
+		"test\"ing",
+		"after",
+		"multitype_array",
+		"should not break with { character in key",
+	}
+	k := o.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// Check nested maps are converted to orderedmaps
+	// nested 1 level deep
+	expectedKeys = []string{
+		"e",
+		"a { nested key with brace",
+		"after",
+	}
+	vi, ok := o.Get("orderedmap")
+	if !ok {
+		t.Error("Missing key for nested map 1 deep")
+	}
+	v := vi.(*Map)
+	k = v.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// nested 2 levels deep
+	expectedKeys = []string{
+		"link",
+	}
+	vi, ok = v.Get("after")
+	if !ok {
+		t.Error("Missing key for nested map 2 deep")
+	}
+	v = vi.(*Map)
+	k = v.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// multitype array
+	expectedKeys = []string{
+		"map",
+		"it",
+		":colon in key",
+	}
+	vislice, ok := o.Get("multitype_array")
+	if !ok {
+		t.Error("Missing key for multitype array")
+	}
+	vslice := vislice.([]interface{})
+	vmap := vslice[2].(*Map)
+	k = vmap.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+}
+
+func TestUnmarshalJSONSpecialChars(t *testing.T) {
+	s := `{ " \\\\\\\\\\\\ "  : { "\\\\\\" : "\\\\\"\\" }, "\\":  " \\\\ test " }`
+	o := New()
+	err := json.Unmarshal([]byte(s), &o)
+	if err != nil {
+		t.Error("JSON Unmarshal error with special chars", err)
+	}
+}
+
+func TestUnmarshalJSONArrayOfMaps(t *testing.T) {
+	s := `
+{
+  "name": "test",
+  "percent": 6,
+  "breakdown": [
+    {
+      "name": "a",
+      "percent": 0.9
+    },
+    {
+      "name": "b",
+      "percent": 0.9
+    },
+    {
+      "name": "d",
+      "percent": 0.4
+    },
+    {
+      "name": "e",
+      "percent": 2.7
+    }
+  ]
+}
+`
+	o := New()
+	err := json.Unmarshal([]byte(s), &o)
+	if err != nil {
+		t.Error("JSON Unmarshal error", err)
+	}
+	// Check the root keys
+	expectedKeys := []string{
+		"name",
+		"percent",
+		"breakdown",
+	}
+	k := o.Keys()
+	for i := range k {
+		if k[i] != expectedKeys[i] {
+			t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i])
+		}
+	}
+	// Check nested maps are converted to orderedmaps
+	// nested 1 level deep
+	expectedKeys = []string{
+		"name",
+		"percent",
+	}
+	vi, ok := o.Get("breakdown")
+	if !ok {
+		t.Error("Missing key for nested map 1 deep")
+	}
+	vs := vi.([]interface{})
+	for _, vInterface := range vs {
+		v := vInterface.(*Map)
+		k = v.Keys()
+		for i := range k {
+			if k[i] != expectedKeys[i] {
+				t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i])
+			}
+		}
+	}
+}
+
+func TestInsertAt(t *testing.T) {
+	om := New()
+	om.Set("zero", 0)
+	om.Set("one", 1)
+	om.Set("two", 2)
+
+	err := om.InsertAt("TEST", 10000, 4) //3 is this added one in size of map
+	if err == nil {
+		t.Error("expected insert at greater position than size of map to produce error")
+	}
+
+	err = om.InsertAt("A", 100, 2)
+	if err != nil {
+		t.Error(err)
+	}
+	// Test it's at end
+	if om.values[om.keys[2]] != 100 {
+		t.Error("expected entry A to be at position 2", om.keys)
+	}
+	if om.values[om.keys[3]] != 2 {
+		t.Error("expected two to be in position 1", om.keys)
+	}
+
+	err = om.InsertAt("B", 200, 0)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if om.values[om.keys[0]] != 200 {
+		t.Error("expected B to be position 0", om.keys)
+	}
+
+	err = om.InsertAt("C", 300, -1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Should show up at the end
+	if om.values[om.keys[len(om.keys)-1]] != 300 {
+		t.Error(fmt.Sprintf("expected C to be in position %d", len(om.keys)-1), om.keys)
+	}
+
+	err = om.InsertAt("D", 400, 1)
+	if err != nil {
+		t.Error(err)
+	}
+
+	if om.values[om.keys[1]] != 400 {
+		t.Error("expceted D to be position 1", om.keys)
+	}
+
+	err = om.InsertAt("F", 600, -8)
+	if err != nil {
+		t.Error(err)
+	}
+	if om.values[om.keys[0]] != 600 {
+		t.Error("expected F to be in position 0", om.keys)
+	}
+}
+
+func TestConcurrency(t *testing.T) {
+	wg := sync.WaitGroup{}
+	type concurrency struct {
+		a string
+		b int
+		c time.Time
+		d bool
+	}
+
+	//Starting Map
+	m := New()
+	m.Set("A", concurrency{"string", 10, time.Now(), true})
+	m.Set("B", concurrency{"string", 10, time.Now(), true})
+	m.Set("C", concurrency{"string", 10, time.Now(), true})
+	m.Set("D", concurrency{"string", 10, time.Now(), true})
+	m.Set("E", concurrency{"string", 10, time.Now(), true})
+	m.Set("F", concurrency{"string", 10, time.Now(), true})
+	m.Set("G", concurrency{"string", 10, time.Now(), true})
+	m.Set("H", concurrency{"string", 10, time.Now(), true})
+	//Inserts
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 50; i++ {
+			wg.Add(1)
+			go func(index int) {
+				defer wg.Done()
+				m.Set("New"+strconv.Itoa(index), concurrency{"string", index, time.Now(), cast.ToBool(index % 2)})
+			}(i)
+		}
+	}()
+	//Reads
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		for i := 0; i < 50; i++ {
+			wg.Add(1)
+			go func(index int) {
+				defer wg.Done()
+				_, _ = m.Get("New" + strconv.Itoa(rand.Intn(99)))
+			}(i)
+		}
+	}()
+	//Marshalling like endpoint
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+
+		for i := 0; i < 50; i++ {
+			wg.Add(1)
+			go func(index int) {
+				defer wg.Done()
+				_, err := m.MarshalJSON()
+				if err != nil {
+					t.Error(err)
+				}
+			}(i)
+		}
+	}()
+
+	wg.Wait()
+
+}