diff --git a/extras/orderedmap/ordered_map.go b/extras/orderedmap/ordered_map.go new file mode 100644 index 0000000..1be9f14 --- /dev/null +++ b/extras/orderedmap/ordered_map.go @@ -0,0 +1,301 @@ +package orderedmap + +// mostly from https://github.com/iancoleman/orderedmap + +import ( + "encoding/json" + "fmt" + "sort" + "strings" + "sync" + + "github.com/lbryio/lbry.go/extras/errors" +) + +type keyIndex struct { + Key string + Index int +} + +type byIndex []keyIndex + +func (a byIndex) Len() int { return len(a) } +func (a byIndex) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a byIndex) Less(i, j int) bool { return a[i].Index < a[j].Index } + +type Map struct { + l sync.RWMutex + keys []string + values map[string]interface{} +} + +func New() *Map { + o := Map{} + o.l = sync.RWMutex{} + o.keys = []string{} + o.values = map[string]interface{}{} + return &o +} + +func (o *Map) Get(key string) (interface{}, bool) { + o.l.RLock() + defer o.l.RUnlock() + val, exists := o.values[key] + return val, exists +} + +func (o *Map) Set(key string, value interface{}) { + o.l.Lock() + defer o.l.Unlock() + _, exists := o.values[key] + if !exists { + o.keys = append(o.keys, key) + } + o.values[key] = value +} + +const outOfRange = "position value %d is outside of the range %d - %d" + +//InsertAt This is a zero based position index: 0,1,2,3..n(from left) OR -1,-2,-3...-n(from right) where -1 is the last place. +func (o *Map) InsertAt(key string, value interface{}, position int) error { + o.l.Lock() + defer o.l.Unlock() + var index = position + if position < 0 { + // support indexing from the back: -1 = last in array. + index += len(o.keys) + 1 + if index < 0 || index > len(o.keys) { + return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1)) + } + } else if index > len(o.keys) { + return errors.Err(fmt.Sprintf(outOfRange, position, len(o.keys), len(o.keys)-1)) + } + _, exists := o.values[key] + if !exists { + // left + key + right + o.keys = append(o.keys[0:index], append([]string{key}, o.keys[index:]...)...) + } + o.values[key] = value + + return nil +} + +func (o *Map) Prepend(key string, value interface{}) { + o.l.Lock() + defer o.l.Unlock() + _, exists := o.values[key] + if !exists { + o.keys = append([]string{key}, o.keys...) + } + o.values[key] = value +} + +func (o *Map) Delete(key string) { + o.l.Lock() + defer o.l.Unlock() + // check key is in use + _, ok := o.values[key] + if !ok { + return + } + // remove from keys + for i, k := range o.keys { + if k == key { + o.keys = append(o.keys[:i], o.keys[i+1:]...) + break + } + } + // remove from values + delete(o.values, key) +} + +func (o *Map) Keys() []string { + o.l.RLock() + defer o.l.RUnlock() + return o.keys +} + +func (o *Map) UnmarshalJSON(b []byte) error { + o.l.Lock() + defer o.l.Unlock() + m := map[string]interface{}{} + if err := json.Unmarshal(b, &m); err != nil { + return errors.Err(err) + } + s := string(b) + mapToOrderedMap(o, s, m) + return nil +} + +func mapToOrderedMap(o *Map, s string, m map[string]interface{}) { + // Get the order of the keys + orderedKeys := []keyIndex{} + for k := range m { + kEscaped := strings.Replace(k, `"`, `\"`, -1) + kQuoted := `"` + kEscaped + `"` + // Find how much content exists before this key. + // If all content from this key and after is replaced with a close + // brace, it should still form a valid json string. + sTrimmed := s + for len(sTrimmed) > 0 { + lastIndex := strings.LastIndex(sTrimmed, kQuoted) + if lastIndex == -1 { + break + } + sTrimmed = sTrimmed[0:lastIndex] + sTrimmed = strings.TrimSpace(sTrimmed) + if len(sTrimmed) > 0 && sTrimmed[len(sTrimmed)-1] == ',' { + sTrimmed = sTrimmed[0 : len(sTrimmed)-1] + } + maybeValidJson := sTrimmed + "}" + testMap := map[string]interface{}{} + err := json.Unmarshal([]byte(maybeValidJson), &testMap) + if err == nil { + // record the position of this key in s + ki := keyIndex{ + Key: k, + Index: len(sTrimmed), + } + orderedKeys = append(orderedKeys, ki) + // shorten the string to get the next key + startOfValueIndex := lastIndex + len(kQuoted) + valueStr := s[startOfValueIndex : len(s)-1] + valueStr = strings.TrimSpace(valueStr) + if len(valueStr) > 0 && valueStr[0] == ':' { + valueStr = valueStr[1:] + } + valueStr = strings.TrimSpace(valueStr) + if valueStr[0] == '{' { + // if the value for this key is a map, convert it to an orderedmap. + // find end of valueStr by removing everything after last } + // until it forms valid json + hasValidJson := false + i := 1 + for i < len(valueStr) && !hasValidJson { + if valueStr[i] != '}' { + i = i + 1 + continue + } + subTestMap := map[string]interface{}{} + testValue := valueStr[0 : i+1] + err = json.Unmarshal([]byte(testValue), &subTestMap) + if err == nil { + hasValidJson = true + valueStr = testValue + break + } + i = i + 1 + } + // convert to orderedmap + if hasValidJson { + mkTyped := m[k].(map[string]interface{}) + oo := &Map{} + mapToOrderedMap(oo, valueStr, mkTyped) + m[k] = oo + } + } else if valueStr[0] == '[' { + // if the value for this key is a []interface, convert any map items to an orderedmap. + // find end of valueStr by removing everything after last ] + // until it forms valid json + hasValidJson := false + i := 1 + for i < len(valueStr) && !hasValidJson { + if valueStr[i] != ']' { + i = i + 1 + continue + } + subTestSlice := []interface{}{} + testValue := valueStr[0 : i+1] + err = json.Unmarshal([]byte(testValue), &subTestSlice) + if err == nil { + hasValidJson = true + valueStr = testValue + break + } + i = i + 1 + } + if hasValidJson { + itemsStr := valueStr[1 : len(valueStr)-1] + // get next item in the slice + itemIndex := 0 + startItem := 0 + endItem := 0 + for endItem < len(itemsStr) { + if itemsStr[endItem] != ',' && endItem < len(itemsStr)-1 { + endItem = endItem + 1 + continue + } + // if this substring compiles to json, it's the next item + possibleItemStr := strings.TrimSpace(itemsStr[startItem:endItem]) + var possibleItem interface{} + err = json.Unmarshal([]byte(possibleItemStr), &possibleItem) + if err != nil { + endItem = endItem + 1 + continue + } + // if item is map, convert to orderedmap + if possibleItemStr[0] == '{' { + mkTyped := m[k].([]interface{}) + mkiTyped := mkTyped[itemIndex].(map[string]interface{}) + oo := &Map{} + mapToOrderedMap(oo, possibleItemStr, mkiTyped) + // replace original map with orderedmap + mkTyped[itemIndex] = oo + m[k] = mkTyped + } + // remove this item from itemsStr + startItem = endItem + 1 + endItem = endItem + 1 + itemIndex = itemIndex + 1 + } + } + } + break + } + } + } + // Sort the keys + sort.Sort(byIndex(orderedKeys)) + // Convert sorted keys to string slice + k := []string{} + for _, ki := range orderedKeys { + k = append(k, ki.Key) + } + // Set the Map values + o.values = m + o.keys = k +} + +func (o *Map) Copy() *Map { + new := New() + + for _, k := range o.keys { + v, _ := o.Get(k) + new.Set(k, v) + } + + return new +} + +func (o *Map) MarshalJSON() ([]byte, error) { + o.l.RLock() + defer o.l.RUnlock() + s := "{" + for _, k := range o.keys { + // add key + kEscaped := strings.Replace(k, `"`, `\"`, -1) + s = s + `"` + kEscaped + `":` + // add value + v := o.values[k] + vBytes, err := json.Marshal(v) + if err != nil { + return []byte{}, errors.Err(err) + } + s = s + string(vBytes) + "," + } + if len(o.keys) > 0 { + s = s[0 : len(s)-1] + } + s = s + "}" + return []byte(s), nil +} diff --git a/extras/orderedmap/ordered_map_test.go b/extras/orderedmap/ordered_map_test.go new file mode 100644 index 0000000..099e091 --- /dev/null +++ b/extras/orderedmap/ordered_map_test.go @@ -0,0 +1,478 @@ +package orderedmap + +import ( + "encoding/json" + "fmt" + "math/rand" + "strconv" + "sync" + "testing" + "time" + + "github.com/spf13/cast" +) + +func TestOrderedMap(t *testing.T) { + o := New() + // number + o.Set("number", 3) + v, _ := o.Get("number") + if v.(int) != 3 { + t.Error("Set number") + } + // string + o.Set("string", "x") + v, _ = o.Get("string") + if v.(string) != "x" { + t.Error("Set string") + } + // string slice + o.Set("strings", []string{ + "t", + "u", + }) + v, _ = o.Get("strings") + if v.([]string)[0] != "t" { + t.Error("Set strings first index") + } + if v.([]string)[1] != "u" { + t.Error("Set strings second index") + } + // mixed slice + o.Set("mixed", []interface{}{ + 1, + "1", + }) + v, _ = o.Get("mixed") + if v.([]interface{})[0].(int) != 1 { + t.Error("Set mixed int") + } + if v.([]interface{})[1].(string) != "1" { + t.Error("Set mixed string") + } + // overriding existing key + o.Set("number", 4) + v, _ = o.Get("number") + if v.(int) != 4 { + t.Error("Override existing key") + } + // Keys method + keys := o.Keys() + expectedKeys := []string{ + "number", + "string", + "strings", + "mixed", + } + for i := range keys { + if keys[i] != expectedKeys[i] { + t.Error("Keys method", keys[i], "!=", expectedKeys[i]) + } + } + for i := range expectedKeys { + if keys[i] != expectedKeys[i] { + t.Error("Keys method", keys[i], "!=", expectedKeys[i]) + } + } + // delete + o.Delete("strings") + o.Delete("not a key being used") + if len(o.Keys()) != 3 { + t.Error("Delete method") + } + _, ok := o.Get("strings") + if ok { + t.Error("Delete did not remove 'strings' key") + } +} + +func TestBlankMarshalJSON(t *testing.T) { + o := New() + // blank map + b, err := json.Marshal(o) + if err != nil { + t.Error("Marshalling blank map to json", err) + } + s := string(b) + // check json is correctly ordered + if s != `{}` { + t.Error("JSON Marshaling blank map value is incorrect", s) + } + // convert to indented json + bi, err := json.MarshalIndent(o, "", " ") + if err != nil { + t.Error("Marshalling indented json for blank map", err) + } + si := string(bi) + ei := `{}` + if si != ei { + fmt.Println(ei) + fmt.Println(si) + t.Error("JSON MarshalIndent blank map value is incorrect", si) + } +} + +func TestMarshalJSON(t *testing.T) { + o := New() + // number + o.Set("number", 3) + // string + o.Set("string", "x") + // new value keeps key in old position + o.Set("number", 4) + // keys not sorted alphabetically + o.Set("z", 1) + o.Set("a", 2) + o.Set("b", 3) + // slice + o.Set("slice", []interface{}{ + "1", + 1, + }) + // orderedmap + v := New() + v.Set("e", 1) + v.Set("a", 2) + o.Set("orderedmap", v) + // double quote in key + o.Set(`test"ing`, 9) + // convert to json + b, err := json.Marshal(o) + if err != nil { + t.Error("Marshalling json", err) + } + s := string(b) + // check json is correctly ordered + if s != `{"number":4,"string":"x","z":1,"a":2,"b":3,"slice":["1",1],"orderedmap":{"e":1,"a":2},"test\"ing":9}` { + t.Error("JSON Marshal value is incorrect", s) + } + // convert to indented json + bi, err := json.MarshalIndent(o, "", " ") + if err != nil { + t.Error("Marshalling indented json", err) + } + si := string(bi) + ei := `{ + "number": 4, + "string": "x", + "z": 1, + "a": 2, + "b": 3, + "slice": [ + "1", + 1 + ], + "orderedmap": { + "e": 1, + "a": 2 + }, + "test\"ing": 9 +}` + if si != ei { + fmt.Println(ei) + fmt.Println(si) + t.Error("JSON MarshalIndent value is incorrect", si) + } +} + +func TestUnmarshalJSON(t *testing.T) { + s := `{ + "number": 4, + "string": "x", + "z": 1, + "a": "should not break with unclosed { character in value", + "b": 3, + "slice": [ + "1", + 1 + ], + "orderedmap": { + "e": 1, + "a { nested key with brace": "with a }}}} }} {{{ brace value", + "after": { + "link": "test {{{ with even deeper nested braces }" + } + }, + "test\"ing": 9, + "after": 1, + "multitype_array": [ + "test", + 1, + { "map": "obj", "it" : 5, ":colon in key": "colon: in value" } + ], + "should not break with { character in key": 1 +}` + o := New() + err := json.Unmarshal([]byte(s), &o) + if err != nil { + t.Error("JSON Unmarshal error", err) + } + // Check the root keys + expectedKeys := []string{ + "number", + "string", + "z", + "a", + "b", + "slice", + "orderedmap", + "test\"ing", + "after", + "multitype_array", + "should not break with { character in key", + } + k := o.Keys() + for i := range k { + if k[i] != expectedKeys[i] { + t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i]) + } + } + // Check nested maps are converted to orderedmaps + // nested 1 level deep + expectedKeys = []string{ + "e", + "a { nested key with brace", + "after", + } + vi, ok := o.Get("orderedmap") + if !ok { + t.Error("Missing key for nested map 1 deep") + } + v := vi.(*Map) + k = v.Keys() + for i := range k { + if k[i] != expectedKeys[i] { + t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i]) + } + } + // nested 2 levels deep + expectedKeys = []string{ + "link", + } + vi, ok = v.Get("after") + if !ok { + t.Error("Missing key for nested map 2 deep") + } + v = vi.(*Map) + k = v.Keys() + for i := range k { + if k[i] != expectedKeys[i] { + t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i]) + } + } + // multitype array + expectedKeys = []string{ + "map", + "it", + ":colon in key", + } + vislice, ok := o.Get("multitype_array") + if !ok { + t.Error("Missing key for multitype array") + } + vslice := vislice.([]interface{}) + vmap := vslice[2].(*Map) + k = vmap.Keys() + for i := range k { + if k[i] != expectedKeys[i] { + t.Error("Key order for nested map 2 deep", i, k[i], "!=", expectedKeys[i]) + } + } +} + +func TestUnmarshalJSONSpecialChars(t *testing.T) { + s := `{ " \\\\\\\\\\\\ " : { "\\\\\\" : "\\\\\"\\" }, "\\": " \\\\ test " }` + o := New() + err := json.Unmarshal([]byte(s), &o) + if err != nil { + t.Error("JSON Unmarshal error with special chars", err) + } +} + +func TestUnmarshalJSONArrayOfMaps(t *testing.T) { + s := ` +{ + "name": "test", + "percent": 6, + "breakdown": [ + { + "name": "a", + "percent": 0.9 + }, + { + "name": "b", + "percent": 0.9 + }, + { + "name": "d", + "percent": 0.4 + }, + { + "name": "e", + "percent": 2.7 + } + ] +} +` + o := New() + err := json.Unmarshal([]byte(s), &o) + if err != nil { + t.Error("JSON Unmarshal error", err) + } + // Check the root keys + expectedKeys := []string{ + "name", + "percent", + "breakdown", + } + k := o.Keys() + for i := range k { + if k[i] != expectedKeys[i] { + t.Error("Unmarshal root key order", i, k[i], "!=", expectedKeys[i]) + } + } + // Check nested maps are converted to orderedmaps + // nested 1 level deep + expectedKeys = []string{ + "name", + "percent", + } + vi, ok := o.Get("breakdown") + if !ok { + t.Error("Missing key for nested map 1 deep") + } + vs := vi.([]interface{}) + for _, vInterface := range vs { + v := vInterface.(*Map) + k = v.Keys() + for i := range k { + if k[i] != expectedKeys[i] { + t.Error("Key order for nested map 1 deep ", i, k[i], "!=", expectedKeys[i]) + } + } + } +} + +func TestInsertAt(t *testing.T) { + om := New() + om.Set("zero", 0) + om.Set("one", 1) + om.Set("two", 2) + + err := om.InsertAt("TEST", 10000, 4) //3 is this added one in size of map + if err == nil { + t.Error("expected insert at greater position than size of map to produce error") + } + + err = om.InsertAt("A", 100, 2) + if err != nil { + t.Error(err) + } + // Test it's at end + if om.values[om.keys[2]] != 100 { + t.Error("expected entry A to be at position 2", om.keys) + } + if om.values[om.keys[3]] != 2 { + t.Error("expected two to be in position 1", om.keys) + } + + err = om.InsertAt("B", 200, 0) + if err != nil { + t.Error(err) + } + + if om.values[om.keys[0]] != 200 { + t.Error("expected B to be position 0", om.keys) + } + + err = om.InsertAt("C", 300, -1) + if err != nil { + t.Error(err) + } + + // Should show up at the end + if om.values[om.keys[len(om.keys)-1]] != 300 { + t.Error(fmt.Sprintf("expected C to be in position %d", len(om.keys)-1), om.keys) + } + + err = om.InsertAt("D", 400, 1) + if err != nil { + t.Error(err) + } + + if om.values[om.keys[1]] != 400 { + t.Error("expceted D to be position 1", om.keys) + } + + err = om.InsertAt("F", 600, -8) + if err != nil { + t.Error(err) + } + if om.values[om.keys[0]] != 600 { + t.Error("expected F to be in position 0", om.keys) + } +} + +func TestConcurrency(t *testing.T) { + wg := sync.WaitGroup{} + type concurrency struct { + a string + b int + c time.Time + d bool + } + + //Starting Map + m := New() + m.Set("A", concurrency{"string", 10, time.Now(), true}) + m.Set("B", concurrency{"string", 10, time.Now(), true}) + m.Set("C", concurrency{"string", 10, time.Now(), true}) + m.Set("D", concurrency{"string", 10, time.Now(), true}) + m.Set("E", concurrency{"string", 10, time.Now(), true}) + m.Set("F", concurrency{"string", 10, time.Now(), true}) + m.Set("G", concurrency{"string", 10, time.Now(), true}) + m.Set("H", concurrency{"string", 10, time.Now(), true}) + //Inserts + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + m.Set("New"+strconv.Itoa(index), concurrency{"string", index, time.Now(), cast.ToBool(index % 2)}) + }(i) + } + }() + //Reads + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + _, _ = m.Get("New" + strconv.Itoa(rand.Intn(99))) + }(i) + } + }() + //Marshalling like endpoint + wg.Add(1) + go func() { + defer wg.Done() + + for i := 0; i < 50; i++ { + wg.Add(1) + go func(index int) { + defer wg.Done() + _, err := m.MarshalJSON() + if err != nil { + t.Error(err) + } + }(i) + } + }() + + wg.Wait() + +}