Add ordered_map to lbry.go for use in other repositories.

This commit is contained in:
Mark Beamer Jr 2019-12-11 20:39:18 -05:00
parent 06764c3d00
commit a391c83a2f
No known key found for this signature in database
GPG key ID: 1C314FB89AD76973
2 changed files with 779 additions and 0 deletions

View file

@ -0,0 +1,301 @@
package orderedmap
// mostly from https://github.com/iancoleman/orderedmap
import (
"encoding/json"
"fmt"
"sort"
"strings"
"sync"
"github.com/lbryio/lbry.go/extras/errors"
)
type keyIndex struct {
Key string
Index int
}
type byIndex []keyIndex
func (a byIndex) Len() int { return len(a) }
func (a byIndex) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a byIndex) Less(i, j int) bool { return a[i].Index < a[j].Index }
type Map struct {
l sync.RWMutex
keys []string
values map[string]interface{}
}
func New() *Map {
o := Map{}
o.l = sync.RWMutex{}
o.keys = []string{}
o.values = map[string]interface{}{}
return &o
}
func (o *Map) Get(key string) (interface{}, bool) {
o.l.RLock()
defer o.l.RUnlock()
val, exists := o.values[key]
return val, exists
}
func (o *Map) Set(key string, value interface{}) {
o.l.Lock()
defer o.l.Unlock()
_, exists := o.values[key]
if !exists {
o.keys = append(o.keys, key)
}
o.values[key] = value
}
const outOfRange = "position value %d is outside of the range %d - %d"
//InsertAt This is a zero based position index: 0,1,2,3..n(from left) OR -1,-2,-3...-n(from right) where -1 is the last place.
func (o *Map) InsertAt(key string, value interface{}, position int) error {
o.l.Lock()
defer o.l.Unlock()
var index = position
if position < 0 {
// support indexing from the back: -1 = last in array.
index += len(o.keys) + 1
if index < 0 || index > len(o.keys) {
return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1))
}
} else if index > len(o.keys) {
return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1))
}
_, exists := o.values[key]
if !exists {
// left + key + right
o.keys = append(o.keys[0:index], append([]string{key}, o.keys[index:]...)...)
}
o.values[key] = value
return nil
}
func (o *Map) Prepend(key string, value interface{}) {
o.l.Lock()
defer o.l.Unlock()
_, exists := o.values[key]
if !exists {
o.keys = append([]string{key}, o.keys...)
}
o.values[key] = value
}
func (o *Map) Delete(key string) {
o.l.Lock()
defer o.l.Unlock()
// check key is in use
_, ok := o.values[key]
if !ok {
return
}
// remove from keys
for i, k := range o.keys {
if k == key {
o.keys = append(o.keys[:i], o.keys[i+1:]...)
break
}
}
// remove from values
delete(o.values, key)
}
func (o *Map) Keys() []string {
o.l.RLock()
defer o.l.RUnlock()
return o.keys
}
func (o *Map) UnmarshalJSON(b []byte) error {
o.l.Lock()
defer o.l.Unlock()
m := map[string]interface{}{}
if err := json.Unmarshal(b, &m); err != nil {
return errors.Err(err)
}
s := string(b)
mapToOrderedMap(o, s, m)
return nil
}
func mapToOrderedMap(o *Map, s string, m map[string]interface{}) {
// Get the order of the keys
orderedKeys := []keyIndex{}
for k := range m {
kEscaped := strings.Replace(k, `"`, `\"`, -1)
kQuoted := `"` + kEscaped + `"`
// Find how much content exists before this key.
// If all content from this key and after is replaced with a close
// brace, it should still form a valid json string.
sTrimmed := s
for len(sTrimmed) > 0 {
lastIndex := strings.LastIndex(sTrimmed, kQuoted)
if lastIndex == -1 {
break
}
sTrimmed = sTrimmed[0:lastIndex]
sTrimmed = strings.TrimSpace(sTrimmed)
if len(sTrimmed) > 0 && sTrimmed[len(sTrimmed)-1] == ',' {
sTrimmed = sTrimmed[0 : len(sTrimmed)-1]
}
maybeValidJson := sTrimmed + "}"
testMap := map[string]interface{}{}
err := json.Unmarshal([]byte(maybeValidJson), &testMap)
if err == nil {
// record the position of this key in s
ki := keyIndex{
Key: k,
Index: len(sTrimmed),
}
orderedKeys = append(orderedKeys, ki)
// shorten the string to get the next key
startOfValueIndex := lastIndex + len(kQuoted)
valueStr := s[startOfValueIndex : len(s)-1]
valueStr = strings.TrimSpace(valueStr)
if len(valueStr) > 0 && valueStr[0] == ':' {
valueStr = valueStr[1:]
}
valueStr = strings.TrimSpace(valueStr)
if valueStr[0] == '{' {
// if the value for this key is a map, convert it to an orderedmap.
// find end of valueStr by removing everything after last }
// until it forms valid json
hasValidJson := false
i := 1
for i < len(valueStr) && !hasValidJson {
if valueStr[i] != '}' {
i = i + 1
continue
}
subTestMap := map[string]interface{}{}
testValue := valueStr[0 : i+1]
err = json.Unmarshal([]byte(testValue), &subTestMap)
if err == nil {
hasValidJson = true
valueStr = testValue
break
}
i = i + 1
}
// convert to orderedmap
if hasValidJson {
mkTyped := m[k].(map[string]interface{})
oo := &Map{}
mapToOrderedMap(oo, valueStr, mkTyped)
m[k] = oo
}
} else if valueStr[0] == '[' {
// if the value for this key is a []interface, convert any map items to an orderedmap.
// find end of valueStr by removing everything after last ]
// until it forms valid json
hasValidJson := false
i := 1
for i < len(valueStr) && !hasValidJson {
if valueStr[i] != ']' {
i = i + 1
continue
}
subTestSlice := []interface{}{}
testValue := valueStr[0 : i+1]
err = json.Unmarshal([]byte(testValue), &subTestSlice)
if err == nil {
hasValidJson = true
valueStr = testValue
break
}
i = i + 1
}
if hasValidJson {
itemsStr := valueStr[1 : len(valueStr)-1]
// get next item in the slice
itemIndex := 0
startItem := 0
endItem := 0
for endItem < len(itemsStr) {
if itemsStr[endItem] != ',' && endItem < len(itemsStr)-1 {
endItem = endItem + 1
continue
}
// if this substring compiles to json, it's the next item
possibleItemStr := strings.TrimSpace(itemsStr[startItem:endItem])
var possibleItem interface{}
err = json.Unmarshal([]byte(possibleItemStr), &possibleItem)
if err != nil {
endItem = endItem + 1
continue
}
// if item is map, convert to orderedmap
if possibleItemStr[0] == '{' {
mkTyped := m[k].([]interface{})
mkiTyped := mkTyped[itemIndex].(map[string]interface{})
oo := &Map{}
mapToOrderedMap(oo, possibleItemStr, mkiTyped)
// replace original map with orderedmap
mkTyped[itemIndex] = oo
m[k] = mkTyped
}
// remove this item from itemsStr
startItem = endItem + 1
endItem = endItem + 1
itemIndex = itemIndex + 1
}
}
}
break
}
}
}
// Sort the keys
sort.Sort(byIndex(orderedKeys))
// Convert sorted keys to string slice
k := []string{}
for _, ki := range orderedKeys {
k = append(k, ki.Key)
}
// Set the Map values
o.values = m
o.keys = k
}
func (o *Map) Copy() *Map {
new := New()
for _, k := range o.keys {
v, _ := o.Get(k)
new.Set(k, v)
}
return new
}
func (o *Map) MarshalJSON() ([]byte, error) {
o.l.RLock()
defer o.l.RUnlock()
s := "{"
for _, k := range o.keys {
// add key
kEscaped := strings.Replace(k, `"`, `\"`, -1)
s = s + `"` + kEscaped + `":`
// add value
v := o.values[k]
vBytes, err := json.Marshal(v)
if err != nil {
return []byte{}, errors.Err(err)
}
s = s + string(vBytes) + ","
}
if len(o.keys) > 0 {
s = s[0 : len(s)-1]
}
s = s + "}"
return []byte(s), nil
}

View file

@ -0,0 +1,478 @@
package orderedmap
import (
"encoding/json"
"fmt"
"math/rand"
"strconv"
"sync"
"testing"
"time"
"github.com/spf13/cast"
)
func TestOrderedMap(t *testing.T) {
o := New()
// number
o.Set("number", 3)
v, _ := o.Get("number")
if v.(int) != 3 {
t.Error("Set number")
}
// string
o.Set("string", "x")
v, _ = o.Get("string")
if v.(string) != "x" {
t.Error("Set string")
}
// string slice
o.Set("strings", []string{
"t",
"u",
})
v, _ = o.Get("strings")
if v.([]string)[0] != "t" {
t.Error("Set strings first index")
}
if v.([]string)[1] != "u" {
t.Error("Set strings second index")
}
// mixed slice
o.Set("mixed", []interface{}{
1,
"1",
})
v, _ = o.Get("mixed")
if v.([]interface{})[0].(int) != 1 {
t.Error("Set mixed int")
}
if v.([]interface{})[1].(string) != "1" {
t.Error("Set mixed string")
}
// overriding existing key
o.Set("number", 4)
v, _ = o.Get("number")
if v.(int) != 4 {
t.Error("Override existing key")
}
// Keys method
keys := o.Keys()
expectedKeys := []string{
"number",
"string",
"strings",
"mixed",
}
for i := range keys {
if keys[i] != expectedKeys[i] {
t.Error("Keys method", keys[i], "!=", expectedKeys[i])
}
}
for i := range expectedKeys {
if keys[i] != expectedKeys[i] {
t.Error("Keys method", keys[i], "!=", expectedKeys[i])
}
}
// delete
o.Delete("strings")
o.Delete("not a key being used")
if len(o.Keys()) != 3 {
t.Error("Delete method")
}
_, ok := o.Get("strings")
if ok {
t.Error("Delete did not remove 'strings' key")
}
}
func TestBlankMarshalJSON(t *testing.T) {
o := New()
// blank map
b, err := json.Marshal(o)
if err != nil {
t.Error("Marshalling blank map to json", err)
}
s := string(b)
// check json is correctly ordered
if s != `{}` {
t.Error("JSON Marshaling blank map value is incorrect", s)
}
// convert to indented json
bi, err := json.MarshalIndent(o, "", " ")
if err != nil {
t.Error("Marshalling indented json for blank map", err)
}
si := string(bi)
ei := `{}`
if si != ei {
fmt.Println(ei)
fmt.Println(si)
t.Error("JSON MarshalIndent blank map value is incorrect", si)
}
}
func TestMarshalJSON(t *testing.T) {
o := New()
// number
o.Set("number", 3)
// string
o.Set("string", "x")
// new value keeps key in old position
o.Set("number", 4)
// keys not sorted alphabetically
o.Set("z", 1)
o.Set("a", 2)
o.Set("b", 3)
// slice
o.Set("slice", []interface{}{
"1",
1,
})
// orderedmap
v := New()
v.Set("e", 1)
v.Set("a", 2)
o.Set("orderedmap", v)
// double quote in key
o.Set(`test"ing`, 9)
// convert to json
b, err := json.Marshal(o)
if err != nil {
t.Error("Marshalling json", err)
}
s := string(b)
// check json is correctly ordered
if s != `{"number":4,"string":"x","z":1,"a":2,"b":3,"slice":["1",1],"orderedmap":{"e":1,"a":2},"test\"ing":9}` {
t.Error("JSON Marshal value is incorrect", s)
}
// convert to indented json
bi, err := json.MarshalIndent(o, "", " ")
if err != nil {
t.Error("Marshalling indented json", err)
}
si := string(bi)
ei := `{
"number": 4,
"string": "x",
"z": 1,
"a": 2,
"b": 3,
"slice": [
"1",
1
],
"orderedmap": {
"e": 1,
"a": 2
},
"test\"ing": 9
}`
if si != ei {
fmt.Println(ei)
fmt.Println(si)
t.Error("JSON MarshalIndent value is incorrect", si)
}
}
func TestUnmarshalJSON(t *testing.T) {
s := `{
"number": 4,
"string": "x",
"z": 1,
"a": "should not break with unclosed { character in value",
"b": 3,
"slice": [
"1",
1
],
"orderedmap": {
"e": 1,
"a { nested key with brace": "with a }}}} }} {{{ brace value",
"after": {
"link": "test {{{ with even deeper nested braces }"
}
},
"test\"ing": 9,
"after": 1,
"multitype_array": [
"test",
1,
{ "map": "obj", "it" : 5, ":colon in key": "colon: in value" }
],
"should not break with { character in key": 1
}`
o := New()
err := json.Unmarshal([]byte(s), &o)
if err != nil {
t.Error("JSON Unmarshal error", err)
}
// Check the root keys
expectedKeys := []string{
"number",
"string",
"z",
"a",
"b",
"slice",
"orderedmap",
"test\"ing",
"after",
"multitype_array",
"should not break with { character in key",
}
k := o.Keys()
for i := range k {
if k[i] != expectedKeys[i] {
t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i])
}
}
// Check nested maps are converted to orderedmaps
// nested 1 level deep
expectedKeys = []string{
"e",
"a { nested key with brace",
"after",
}
vi, ok := o.Get("orderedmap")
if !ok {
t.Error("Missing key for nested map 1 deep")
}
v := vi.(*Map)
k = v.Keys()
for i := range k {
if k[i] != expectedKeys[i] {
t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i])
}
}
// nested 2 levels deep
expectedKeys = []string{
"link",
}
vi, ok = v.Get("after")
if !ok {
t.Error("Missing key for nested map 2 deep")
}
v = vi.(*Map)
k = v.Keys()
for i := range k {
if k[i] != expectedKeys[i] {
t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i])
}
}
// multitype array
expectedKeys = []string{
"map",
"it",
":colon in key",
}
vislice, ok := o.Get("multitype_array")
if !ok {
t.Error("Missing key for multitype array")
}
vslice := vislice.([]interface{})
vmap := vslice[2].(*Map)
k = vmap.Keys()
for i := range k {
if k[i] != expectedKeys[i] {
t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i])
}
}
}
func TestUnmarshalJSONSpecialChars(t *testing.T) {
s := `{ " \\\\\\\\\\\\ " : { "\\\\\\" : "\\\\\"\\" }, "\\": " \\\\ test " }`
o := New()
err := json.Unmarshal([]byte(s), &o)
if err != nil {
t.Error("JSON Unmarshal error with special chars", err)
}
}
func TestUnmarshalJSONArrayOfMaps(t *testing.T) {
s := `
{
"name": "test",
"percent": 6,
"breakdown": [
{
"name": "a",
"percent": 0.9
},
{
"name": "b",
"percent": 0.9
},
{
"name": "d",
"percent": 0.4
},
{
"name": "e",
"percent": 2.7
}
]
}
`
o := New()
err := json.Unmarshal([]byte(s), &o)
if err != nil {
t.Error("JSON Unmarshal error", err)
}
// Check the root keys
expectedKeys := []string{
"name",
"percent",
"breakdown",
}
k := o.Keys()
for i := range k {
if k[i] != expectedKeys[i] {
t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i])
}
}
// Check nested maps are converted to orderedmaps
// nested 1 level deep
expectedKeys = []string{
"name",
"percent",
}
vi, ok := o.Get("breakdown")
if !ok {
t.Error("Missing key for nested map 1 deep")
}
vs := vi.([]interface{})
for _, vInterface := range vs {
v := vInterface.(*Map)
k = v.Keys()
for i := range k {
if k[i] != expectedKeys[i] {
t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i])
}
}
}
}
func TestInsertAt(t *testing.T) {
om := New()
om.Set("zero", 0)
om.Set("one", 1)
om.Set("two", 2)
err := om.InsertAt("TEST", 10000, 4) //3 is this added one in size of map
if err == nil {
t.Error("expected insert at greater position than size of map to produce error")
}
err = om.InsertAt("A", 100, 2)
if err != nil {
t.Error(err)
}
// Test it's at end
if om.values[om.keys[2]] != 100 {
t.Error("expected entry A to be at position 2", om.keys)
}
if om.values[om.keys[3]] != 2 {
t.Error("expected two to be in position 1", om.keys)
}
err = om.InsertAt("B", 200, 0)
if err != nil {
t.Error(err)
}
if om.values[om.keys[0]] != 200 {
t.Error("expected B to be position 0", om.keys)
}
err = om.InsertAt("C", 300, -1)
if err != nil {
t.Error(err)
}
// Should show up at the end
if om.values[om.keys[len(om.keys)-1]] != 300 {
t.Error(fmt.Sprintf("expected C to be in position %d", len(om.keys)-1), om.keys)
}
err = om.InsertAt("D", 400, 1)
if err != nil {
t.Error(err)
}
if om.values[om.keys[1]] != 400 {
t.Error("expceted D to be position 1", om.keys)
}
err = om.InsertAt("F", 600, -8)
if err != nil {
t.Error(err)
}
if om.values[om.keys[0]] != 600 {
t.Error("expected F to be in position 0", om.keys)
}
}
func TestConcurrency(t *testing.T) {
wg := sync.WaitGroup{}
type concurrency struct {
a string
b int
c time.Time
d bool
}
//Starting Map
m := New()
m.Set("A", concurrency{"string", 10, time.Now(), true})
m.Set("B", concurrency{"string", 10, time.Now(), true})
m.Set("C", concurrency{"string", 10, time.Now(), true})
m.Set("D", concurrency{"string", 10, time.Now(), true})
m.Set("E", concurrency{"string", 10, time.Now(), true})
m.Set("F", concurrency{"string", 10, time.Now(), true})
m.Set("G", concurrency{"string", 10, time.Now(), true})
m.Set("H", concurrency{"string", 10, time.Now(), true})
//Inserts
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 50; i++ {
wg.Add(1)
go func(index int) {
defer wg.Done()
m.Set("New"+strconv.Itoa(index), concurrency{"string", index, time.Now(), cast.ToBool(index % 2)})
}(i)
}
}()
//Reads
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 50; i++ {
wg.Add(1)
go func(index int) {
defer wg.Done()
_, _ = m.Get("New" + strconv.Itoa(rand.Intn(99)))
}(i)
}
}()
//Marshalling like endpoint
wg.Add(1)
go func() {
defer wg.Done()
for i := 0; i < 50; i++ {
wg.Add(1)
go func(index int) {
defer wg.Done()
_, err := m.MarshalJSON()
if err != nil {
t.Error(err)
}
}(i)
}
}()
wg.Wait()
}