Replace bleve gtreap store with a more compact implementation. (08f4fe05) · Commits · public-projects / Nakama

go.mod

+2 −0

Original line number	Diff line number	Diff line
		@@ -4,6 +4,7 @@ go 1.16

		require (
		github.com/blevesearch/bleve/v2 v2.0.3
		github.com/blevesearch/upsidedown_store_api v1.0.1
		github.com/cockroachdb/apd v1.1.0 // indirect
		github.com/dgrijalva/jwt-go v3.2.1-0.20200107013213-dc14462fd587+incompatible
		github.com/dop251/goja v0.0.0-20210406175830-1b11a6af686d
		@@ -23,6 +24,7 @@ require (
		github.com/rubenv/sql-migrate v0.0.0-20210408115534-a32ed26c37ea
		github.com/satori/go.uuid v1.2.0 // indirect
		github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24 // indirect
		github.com/steveyen/gtreap v0.1.0
		github.com/stretchr/testify v1.7.0
		github.com/tinylib/msgp v1.1.2 // indirect
		github.com/uber-go/tally v3.3.17+incompatible

go.sum

+0 −2

Original line number	Diff line number	Diff line
		@@ -256,8 +256,6 @@ github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO
		github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
		github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
		github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
		github.com/heroiclabs/nakama-common v1.13.2-0.20210430100819-3c41e7f99ed1 h1:FM80Pm+cjzzJwaKVjVpjv7QKzfvAMBbT51uwKS8FwmE=
		github.com/heroiclabs/nakama-common v1.13.2-0.20210430100819-3c41e7f99ed1/go.mod h1:jzIGV5bI45ALRQFzHPkJn4Z0tV+xhtho1+pZhOXVAsk=
		github.com/heroiclabs/nakama-common v1.14.0 h1:34b9b4xpqDDIOz236CKJwcuINYsHBqfgveQYs2gYLsE=
		github.com/heroiclabs/nakama-common v1.14.0/go.mod h1:jzIGV5bI45ALRQFzHPkJn4Z0tV+xhtho1+pZhOXVAsk=
		github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=

gtreap_compact/iterator.go

0 → 100644

+152 −0

Original line number	Diff line number	Diff line
		// Copyright (c) 2015 Couchbase, Inc.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		// Package gtreap provides an in-memory implementation of the
		// KVStore interfaces using the gtreap balanced-binary treap,
		// copy-on-write data structure.
		package gtreap_compact

		import (
		"bytes"
		"sync"

		"github.com/steveyen/gtreap"
		)

		type Iterator struct {
		t *gtreap.Treap

		m sync.Mutex
		cancelCh chan struct{}
		nextCh chan *Item
		curr *Item
		currOk bool

		prefix []byte
		start []byte
		end []byte
		}

		func (w *Iterator) Seek(k []byte) {
		if w.start != nil && bytes.Compare(k, w.start) < 0 {
		k = w.start
		}
		if w.prefix != nil && !bytes.HasPrefix(k, w.prefix) {
		if bytes.Compare(k, w.prefix) < 0 {
		k = w.prefix
		} else {
		var end []byte
		for i := len(w.prefix) - 1; i >= 0; i-- {
		c := w.prefix[i]
		if c < 0xff {
		end = make([]byte, i+1)
		copy(end, w.prefix)
		end[i] = c + 1
		break
		}
		}
		k = end
		}
		}
		w.restart(&Item{k: k})
		}

		func (w Iterator) restart(start Item) *Iterator {
		cancelCh := make(chan struct{})
		nextCh := make(chan *Item, 1)

		w.m.Lock()
		if w.cancelCh != nil {
		close(w.cancelCh)
		}
		w.cancelCh = cancelCh
		w.nextCh = nextCh
		w.curr = nil
		w.currOk = false
		w.m.Unlock()

		go func() {
		if start != nil {
		w.t.VisitAscend(start, func(itm gtreap.Item) bool {
		select {
		case <-cancelCh:
		return false
		case nextCh <- itm.(*Item):
		return true
		}
		})
		}
		close(nextCh)
		}()

		w.Next()

		return w
		}

		func (w *Iterator) Next() {
		w.m.Lock()
		nextCh := w.nextCh
		w.m.Unlock()
		w.curr, w.currOk = <-nextCh
		}

		func (w *Iterator) Current() ([]byte, []byte, bool) {
		w.m.Lock()
		defer w.m.Unlock()
		if !w.currOk \|\| w.curr == nil {
		return nil, nil, false
		}
		if w.prefix != nil && !bytes.HasPrefix(w.curr.k, w.prefix) {
		return nil, nil, false
		} else if w.end != nil && bytes.Compare(w.curr.k, w.end) >= 0 {
		return nil, nil, false
		}
		return w.curr.k, w.curr.v, w.currOk
		}

		func (w *Iterator) Key() []byte {
		k, _, ok := w.Current()
		if !ok {
		return nil
		}
		return k
		}

		func (w *Iterator) Value() []byte {
		_, v, ok := w.Current()
		if !ok {
		return nil
		}
		return v
		}

		func (w *Iterator) Valid() bool {
		_, _, ok := w.Current()
		return ok
		}

		func (w *Iterator) Close() error {
		w.m.Lock()
		if w.cancelCh != nil {
		close(w.cancelCh)
		}
		w.cancelCh = nil
		w.nextCh = nil
		w.curr = nil
		w.currOk = false
		w.m.Unlock()

		return nil
		}

gtreap_compact/reader.go

0 → 100644

+66 −0

Original line number	Diff line number	Diff line
		// Copyright (c) 2015 Couchbase, Inc.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		// Package gtreap provides an in-memory implementation of the
		// KVStore interfaces using the gtreap balanced-binary treap,
		// copy-on-write data structure.
		package gtreap_compact

		import (
		"github.com/blevesearch/upsidedown_store_api"

		"github.com/steveyen/gtreap"
		)

		type Reader struct {
		t *gtreap.Treap
		}

		func (r *Reader) Get(k []byte) (v []byte, err error) {
		var rv []byte
		itm := r.t.Get(&Item{k: k})
		if itm != nil {
		rv = make([]byte, len(itm.(*Item).v))
		copy(rv, itm.(*Item).v)
		return rv, nil
		}
		return nil, nil
		}

		func (r *Reader) MultiGet(keys [][]byte) ([][]byte, error) {
		return store.MultiGet(r, keys)
		}

		func (r *Reader) PrefixIterator(k []byte) store.KVIterator {
		rv := Iterator{
		t: r.t,
		prefix: k,
		}
		rv.restart(&Item{k: k})
		return &rv
		}

		func (r *Reader) RangeIterator(start, end []byte) store.KVIterator {
		rv := Iterator{
		t: r.t,
		start: start,
		end: end,
		}
		rv.restart(&Item{k: start})
		return &rv
		}

		func (r *Reader) Close() error {
		return nil
		}

gtreap_compact/store.go

0 → 100644

+118 −0

Original line number	Diff line number	Diff line
		// Copyright (c) 2015 Couchbase, Inc.
		//
		// Licensed under the Apache License, Version 2.0 (the "License");
		// you may not use this file except in compliance with the License.
		// You may obtain a copy of the License at
		//
		// http://www.apache.org/licenses/LICENSE-2.0
		//
		// Unless required by applicable law or agreed to in writing, software
		// distributed under the License is distributed on an "AS IS" BASIS,
		// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
		// See the License for the specific language governing permissions and
		// limitations under the License.

		// Package gtreap provides an in-memory implementation of the
		// KVStore interfaces using the gtreap balanced-binary treap,
		// copy-on-write data structure.

		package gtreap_compact

		import (
		"bytes"
		"fmt"
		"os"
		"sync"

		"github.com/blevesearch/bleve/v2/registry"
		"github.com/blevesearch/upsidedown_store_api"
		"github.com/steveyen/gtreap"
		)

		const Name = "gtreap_compact"

		type Store struct {
		m sync.Mutex
		t *gtreap.Treap
		mo store.MergeOperator
		}

		type Item struct {
		k []byte
		v []byte
		}

		func itemCompare(a, b interface{}) int {
		return bytes.Compare(a.(Item).k, b.(Item).k)
		}

		func New(mo store.MergeOperator, config map[string]interface{}) (store.KVStore, error) {
		path, ok := config["path"].(string)
		if !ok {
		return nil, fmt.Errorf("must specify path")
		}
		if path != "" {
		return nil, os.ErrInvalid
		}

		rv := Store{
		t: gtreap.NewTreap(itemCompare),
		mo: mo,
		}
		return &rv, nil
		}

		func (s *Store) Close() error {
		return nil
		}

		func (s *Store) Reader() (store.KVReader, error) {
		s.m.Lock()
		t := s.t
		s.m.Unlock()
		return &Reader{t: t}, nil
		}

		func (s *Store) Writer() (store.KVWriter, error) {
		return &Writer{s: s}, nil
		}

		// Compact removes DictionaryTerm entries with a count of zero.
		// This is a workaround for github issue #374.
		// Code from https://github.com/blevesearch/bleve/pull/1317.
		func (s *Store) Compact() (err error) {
		kvreader, err := s.Reader()
		if err != nil {
		return err
		}

		defer func() {
		if cerr := kvreader.Close(); err == nil && cerr != nil {
		err = cerr
		}
		}()

		prefix := []byte("d")

		s.m.Lock()
		defer s.m.Unlock()
		it := kvreader.PrefixIterator(prefix)
		defer func() {
		if cerr := it.Close(); err == nil && cerr != nil {
		err = cerr
		}
		}()

		for ; it.Valid(); it.Next() {
		k, v, _ := it.Current()
		if bytes.Equal(v, []byte{0}) {
		s.t = s.t.Delete(&Item{k: k})
		}
		}

		return
		}

		func init() {
		registry.RegisterKVStore(Name, New)
		}