Loading CHANGELOG.md +1 −0 Original line number Diff line number Diff line Loading @@ -6,6 +6,7 @@ The format is based on [keep a changelog](http://keepachangelog.com) and this pr ## [Unreleased] ### Changed - RPC now allows omitting the `unwrap` parameter for requests with empty payloads. - Upgrade GRPC dependency. ## [3.18.0] - 2023-10-24 ### Added Loading go.mod +2 −2 Original line number Diff line number Diff line Loading @@ -3,6 +3,7 @@ module github.com/heroiclabs/nakama/v3 go 1.20 require ( github.com/bits-and-blooms/bitset v1.5.0 github.com/blugelabs/bluge v0.2.2 github.com/blugelabs/bluge_segment_api v0.2.0 github.com/blugelabs/query_string v0.3.0 Loading Loading @@ -30,7 +31,7 @@ require ( golang.org/x/crypto v0.11.0 golang.org/x/oauth2 v0.10.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc google.golang.org/grpc v1.56.2 google.golang.org/grpc v1.57.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0-20230206193814-4cb27fcfbb0f Loading @@ -43,7 +44,6 @@ require ( github.com/RoaringBitmap/roaring v0.9.4 // indirect github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect github.com/blevesearch/segment v0.9.0 // indirect Loading go.sum +4 −3 Original line number Diff line number Diff line Loading @@ -72,8 +72,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= Loading Loading @@ -968,8 +969,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= Loading main.go +1 −0 Original line number Diff line number Diff line Loading @@ -39,6 +39,7 @@ import ( "go.uber.org/zap/zapcore" "google.golang.org/protobuf/encoding/protojson" _ "github.com/bits-and-blooms/bitset" _ "github.com/go-gorp/gorp/v3" _ "github.com/golang/snappy" _ "github.com/prometheus/client_golang/prometheus" Loading vendor/github.com/bits-and-blooms/bitset/bitset.go +149 −82 Original line number Diff line number Diff line Loading @@ -33,7 +33,6 @@ Example use: As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. */ package bitset Loading Loading @@ -87,9 +86,20 @@ func (b *BitSet) safeSet() []uint64 { return b.set } // SetBitsetFrom fills the bitset with an array of integers without creating a new BitSet instance func (b *BitSet) SetBitsetFrom(buf []uint64) { b.length = uint(len(buf)) * 64 b.set = buf } // From is a constructor used to create a BitSet from an array of integers func From(buf []uint64) *BitSet { return &BitSet{uint(len(buf)) * 64, buf} return FromWithLength(uint(len(buf))*64, buf) } // FromWithLength constructs from an array of integers and length. func FromWithLength(len uint, set []uint64) *BitSet { return &BitSet{len, set} } // Bytes returns the bitset as array of integers Loading @@ -105,6 +115,17 @@ func wordsNeeded(i uint) int { return int((i + (wordSize - 1)) >> log2WordSize) } // wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. // This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing bitmap). func wordsNeededUnbound(i uint) int { return int((i + (wordSize - 1)) >> log2WordSize) } // wordsIndex calculates the index of words in a `uint64` func wordsIndex(i uint) uint { return i & (wordSize - 1) } // New creates a new BitSet with a hint that length bits will be required func New(length uint) (bset *BitSet) { defer func() { Loading Loading @@ -135,9 +156,8 @@ func (b *BitSet) Len() uint { return b.length } // extendSetMaybe adds additional words to incorporate new bits if needed func (b *BitSet) extendSetMaybe(i uint) { if i >= b.length { // if we need more bits, make 'em // extendSet adds additional words to incorporate new bits if needed func (b *BitSet) extendSet(i uint) { if i >= Cap() { panic("You are exceeding the capacity") } Loading @@ -153,14 +173,13 @@ func (b *BitSet) extendSetMaybe(i uint) { } b.length = i + 1 } } // Test whether bit i is set. func (b *BitSet) Test(i uint) bool { if i >= b.length { return false } return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 return b.set[i>>log2WordSize]&(1<<wordsIndex(i)) != 0 } // Set bit i to 1, the capacity of the bitset is automatically Loading @@ -170,8 +189,10 @@ func (b *BitSet) Test(i uint) bool { // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. func (b *BitSet) Set(i uint) *BitSet { b.extendSetMaybe(i) b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) if i >= b.length { // if we need more bits, make 'em b.extendSet(i) } b.set[i>>log2WordSize] |= 1 << wordsIndex(i) return b } Loading @@ -180,7 +201,7 @@ func (b *BitSet) Clear(i uint) *BitSet { if i >= b.length { return b } b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) b.set[i>>log2WordSize] &^= 1 << wordsIndex(i) return b } Loading @@ -205,7 +226,7 @@ func (b *BitSet) Flip(i uint) *BitSet { if i >= b.length { return b.Set(i) } b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) b.set[i>>log2WordSize] ^= 1 << wordsIndex(i) return b } Loading @@ -218,15 +239,18 @@ func (b *BitSet) FlipRange(start, end uint) *BitSet { if start >= end { return b } b.extendSetMaybe(end - 1) if end-1 >= b.length { // if we need more bits, make 'em b.extendSet(end - 1) } var startWord uint = start >> log2WordSize var endWord uint = end >> log2WordSize b.set[startWord] ^= ^(^uint64(0) << (start & (wordSize - 1))) b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) for i := startWord; i < endWord; i++ { b.set[i] = ^b.set[i] } b.set[endWord] ^= ^uint64(0) >> (-end & (wordSize - 1)) if end&(wordSize-1) != 0 { b.set[endWord] ^= ^uint64(0) >> wordsIndex(-end) } return b } Loading Loading @@ -254,7 +278,10 @@ func (b *BitSet) Shrink(lastbitindex uint) *BitSet { copy(shrunk, b.set[:idx]) b.set = shrunk b.length = length b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1)))) lastWordUsedBits := length % 64 if lastWordUsedBits != 0 { b.set[idx-1] &= allBits >> uint64(64-wordsIndex(lastWordUsedBits)) } return b } Loading Loading @@ -283,7 +310,7 @@ func (b *BitSet) Compact() *BitSet { // this method could be extremely slow and in some cases might cause the entire BitSet // to be recopied. func (b *BitSet) InsertAt(idx uint) *BitSet { insertAtElement := (idx >> log2WordSize) insertAtElement := idx >> log2WordSize // if length of set is a multiple of wordSize we need to allocate more space first if b.isLenExactMultiple() { Loading @@ -302,13 +329,13 @@ func (b *BitSet) InsertAt(idx uint) *BitSet { // generate a mask to extract the data that we need to shift left // within the element where we insert a bit dataMask := ^(uint64(1)<<uint64(idx&(wordSize-1)) - 1) dataMask := uint64(1)<<uint64(wordsIndex(idx)) - 1 // extract that data that we'll shift data := b.set[i] & dataMask data := b.set[i] & (^dataMask) // set the positions of the data mask to 0 in the element where we insert b.set[i] &= ^dataMask b.set[i] &= dataMask // shift data mask to the left and insert its data to the slice element b.set[i] |= data << 1 Loading Loading @@ -356,7 +383,7 @@ func (b *BitSet) DeleteAt(i uint) *BitSet { // generate a mask for the data that needs to be shifted right // within that slice element that gets modified dataMask := ^((uint64(1) << (i & (wordSize - 1))) - 1) dataMask := ^((uint64(1) << wordsIndex(i)) - 1) // extract the data that we'll shift right from the slice element data := b.set[deleteAtElement] & dataMask Loading Loading @@ -394,7 +421,7 @@ func (b *BitSet) NextSet(i uint) (uint, bool) { return 0, false } w := b.set[x] w = w >> (i & (wordSize - 1)) w = w >> wordsIndex(i) if w != 0 { return i + trailingZeroes64(w), true } Loading Loading @@ -423,7 +450,6 @@ func (b *BitSet) NextSet(i uint) (uint, bool) { // j += 1 // } // // // It is possible to retrieve all set bits as follow: // // indices := make([]uint, bitmap.Count()) Loading @@ -438,7 +464,7 @@ func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { if x >= len(b.set) || capacity == 0 { return 0, myanswer[:0] } skip := i & (wordSize - 1) skip := wordsIndex(i) word := b.set[x] >> skip myanswer = myanswer[:capacity] size := int(0) Loading Loading @@ -481,8 +507,8 @@ func (b *BitSet) NextClear(i uint) (uint, bool) { return 0, false } w := b.set[x] w = w >> (i & (wordSize - 1)) wA := allBits >> (i & (wordSize - 1)) w = w >> wordsIndex(i) wA := allBits >> wordsIndex(i) index := i + trailingZeroes64(^w) if w != wA && index < b.length { return index, true Loading Loading @@ -510,7 +536,7 @@ func (b *BitSet) ClearAll() *BitSet { // wordCount returns the number of words used in a bit set func (b *BitSet) wordCount() int { return len(b.set) return wordsNeededUnbound(b.length) } // Clone this BitSet Loading @@ -522,9 +548,10 @@ func (b *BitSet) Clone() *BitSet { return c } // Copy into a destination BitSet // Returning the size of the destination BitSet // like array copy // Copy into a destination BitSet using the Go array copy semantics: // the number of bits copied is the minimum of the number of bits in the current // BitSet (Len()) and the destination Bitset. // We return the number of bits copied in the destination BitSet. func (b *BitSet) Copy(c *BitSet) (count uint) { if c == nil { return Loading @@ -536,8 +563,32 @@ func (b *BitSet) Copy(c *BitSet) (count uint) { if b.length < c.length { count = b.length } // Cleaning the last word is needed to keep the invariant that other functions, such as Count, require // that any bits in the last word that would exceed the length of the bitmask are set to 0. c.cleanLastWord() return } // CopyFull copies into a destination BitSet such that the destination is // identical to the source after the operation, allocating memory if necessary. func (b *BitSet) CopyFull(c *BitSet) { if c == nil { return } c.length = b.length if len(b.set) == 0 { if c.set != nil { c.set = c.set[:0] } } else { if cap(c.set) < len(b.set) { c.set = make([]uint64, len(b.set)) } else { c.set = c.set[:len(b.set)] } copy(c.set, b.set) } } // Count (number of set bits). // Also known as "popcount" or "population count". Loading @@ -561,10 +612,9 @@ func (b *BitSet) Equal(c *BitSet) bool { if b.length == 0 { // if they have both length == 0, then could have nil set return true } // testing for equality shoud not transform the bitset (no call to safeSet) for p, v := range b.set { if c.set[p] != v { wn := b.wordCount() for p:= 0; p < wn; p++ { if c.set[p] != b.set[p] { return false } } Loading Loading @@ -671,7 +721,9 @@ func (b *BitSet) InPlaceIntersection(compare *BitSet) { b.set[i] = 0 } if compare.length > 0 { b.extendSetMaybe(compare.length - 1) if compare.length-1 >= b.length { b.extendSet(compare.length - 1) } } } Loading Loading @@ -710,8 +762,8 @@ func (b *BitSet) InPlaceUnion(compare *BitSet) { if l > int(b.wordCount()) { l = int(b.wordCount()) } if compare.length > 0 { b.extendSetMaybe(compare.length - 1) if compare.length > 0 && compare.length-1 >= b.length { b.extendSet(compare.length - 1) } for i := 0; i < l; i++ { b.set[i] |= compare.set[i] Loading Loading @@ -758,8 +810,8 @@ func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { if l > int(b.wordCount()) { l = int(b.wordCount()) } if compare.length > 0 { b.extendSetMaybe(compare.length - 1) if compare.length > 0 && compare.length-1 >= b.length { b.extendSet(compare.length - 1) } for i := 0; i < l; i++ { b.set[i] ^= compare.set[i] Loading @@ -773,17 +825,17 @@ func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { // Is the length an exact multiple of word sizes? func (b *BitSet) isLenExactMultiple() bool { return b.length%wordSize == 0 return wordsIndex(b.length) == 0 } // Clean last word by setting unused bits to 0 func (b *BitSet) cleanLastWord() { if !b.isLenExactMultiple() { b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize) b.set[len(b.set)-1] &= allBits >> (wordSize - wordsIndex(b.length)) } } // Complement computes the (local) complement of a biset (up to length bits) // Complement computes the (local) complement of a bitset (up to length bits) func (b *BitSet) Complement() (result *BitSet) { panicIfNull(b) result = New(b.length) Loading Loading @@ -811,7 +863,6 @@ func (b *BitSet) None() bool { return false } } return true } return true } Loading Loading @@ -852,7 +903,8 @@ func (b *BitSet) DumpAsBits() string { // BinaryStorageSize returns the binary storage requirements func (b *BitSet) BinaryStorageSize() int { return binary.Size(uint64(0)) + binary.Size(b.set) nWords := b.wordCount() return binary.Size(uint64(0)) + binary.Size(b.set[:nWords]) } // WriteTo writes a BitSet to a stream Loading @@ -866,7 +918,19 @@ func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { } // Write set err = binary.Write(stream, binaryOrder, b.set) // current implementation of bufio.Writer is more memory efficient than // binary.Write for large set writer := bufio.NewWriter(stream) var item = make([]byte, binary.Size(uint64(0))) // for serializing one uint64 nWords := b.wordCount() for i := range b.set[:nWords] { binaryOrder.PutUint64(item, b.set[i]) if nn, err := writer.Write(item); err != nil { return int64(i*binary.Size(uint64(0)) + nn), err } } err = writer.Flush() return int64(b.BinaryStorageSize()), err } Loading @@ -877,6 +941,9 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { // Read length first err := binary.Read(stream, binaryOrder, &length) if err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } return 0, err } newset := New(uint(length)) Loading @@ -885,11 +952,18 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { return 0, errors.New("unmarshalling error: type mismatch") } // Read remaining bytes as set err = binary.Read(stream, binaryOrder, newset.set) if err != nil { var item [8]byte nWords := wordsNeeded(uint(length)) reader := bufio.NewReader(io.LimitReader(stream, 8*int64(nWords))) for i := 0; i < nWords; i++ { if _, err := io.ReadFull(reader, item[:]); err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } return 0, err } newset.set[i] = binaryOrder.Uint64(item[:]) } *b = *newset return int64(b.BinaryStorageSize()), nil Loading @@ -898,25 +972,18 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { // MarshalBinary encodes a BitSet into a binary form and returns the result. func (b *BitSet) MarshalBinary() ([]byte, error) { var buf bytes.Buffer writer := bufio.NewWriter(&buf) _, err := b.WriteTo(writer) _, err := b.WriteTo(&buf) if err != nil { return []byte{}, err } err = writer.Flush() return buf.Bytes(), err } // UnmarshalBinary decodes the binary form generated by MarshalBinary. func (b *BitSet) UnmarshalBinary(data []byte) error { buf := bytes.NewReader(data) reader := bufio.NewReader(buf) _, err := b.ReadFrom(reader) _, err := b.ReadFrom(buf) return err } Loading Loading
CHANGELOG.md +1 −0 Original line number Diff line number Diff line Loading @@ -6,6 +6,7 @@ The format is based on [keep a changelog](http://keepachangelog.com) and this pr ## [Unreleased] ### Changed - RPC now allows omitting the `unwrap` parameter for requests with empty payloads. - Upgrade GRPC dependency. ## [3.18.0] - 2023-10-24 ### Added Loading
go.mod +2 −2 Original line number Diff line number Diff line Loading @@ -3,6 +3,7 @@ module github.com/heroiclabs/nakama/v3 go 1.20 require ( github.com/bits-and-blooms/bitset v1.5.0 github.com/blugelabs/bluge v0.2.2 github.com/blugelabs/bluge_segment_api v0.2.0 github.com/blugelabs/query_string v0.3.0 Loading Loading @@ -30,7 +31,7 @@ require ( golang.org/x/crypto v0.11.0 golang.org/x/oauth2 v0.10.0 google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc google.golang.org/grpc v1.56.2 google.golang.org/grpc v1.57.0 google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.0.0-20230206193814-4cb27fcfbb0f Loading @@ -43,7 +44,6 @@ require ( github.com/RoaringBitmap/roaring v0.9.4 // indirect github.com/axiomhq/hyperloglog v0.0.0-20191112132149-a4c4c47bc57f // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.2.0 // indirect github.com/blevesearch/go-porterstemmer v1.0.3 // indirect github.com/blevesearch/mmap-go v1.0.4 // indirect github.com/blevesearch/segment v0.9.0 // indirect Loading
go.sum +4 −3 Original line number Diff line number Diff line Loading @@ -72,8 +72,9 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/blevesearch/go-porterstemmer v1.0.3 h1:GtmsqID0aZdCSNiY8SkuPJ12pD4jI+DdXTAn4YRcHCo= github.com/blevesearch/go-porterstemmer v1.0.3/go.mod h1:angGc5Ht+k2xhJdZi511LtmxuEf0OVpvUUNrwmM1P7M= Loading Loading @@ -968,8 +969,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0 h1:rNBFJjBCOgVr9pWD7rs/knKL4FRTKgpZmsRfV214zcA= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.3.0/go.mod h1:Dk1tviKTvMCz5tvh7t+fh94dhmQVHuCt2OzJB3CTW9Y= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= Loading
main.go +1 −0 Original line number Diff line number Diff line Loading @@ -39,6 +39,7 @@ import ( "go.uber.org/zap/zapcore" "google.golang.org/protobuf/encoding/protojson" _ "github.com/bits-and-blooms/bitset" _ "github.com/go-gorp/gorp/v3" _ "github.com/golang/snappy" _ "github.com/prometheus/client_golang/prometheus" Loading
vendor/github.com/bits-and-blooms/bitset/bitset.go +149 −82 Original line number Diff line number Diff line Loading @@ -33,7 +33,6 @@ Example use: As an alternative to BitSets, one should check out the 'big' package, which provides a (less set-theoretical) view of bitsets. */ package bitset Loading Loading @@ -87,9 +86,20 @@ func (b *BitSet) safeSet() []uint64 { return b.set } // SetBitsetFrom fills the bitset with an array of integers without creating a new BitSet instance func (b *BitSet) SetBitsetFrom(buf []uint64) { b.length = uint(len(buf)) * 64 b.set = buf } // From is a constructor used to create a BitSet from an array of integers func From(buf []uint64) *BitSet { return &BitSet{uint(len(buf)) * 64, buf} return FromWithLength(uint(len(buf))*64, buf) } // FromWithLength constructs from an array of integers and length. func FromWithLength(len uint, set []uint64) *BitSet { return &BitSet{len, set} } // Bytes returns the bitset as array of integers Loading @@ -105,6 +115,17 @@ func wordsNeeded(i uint) int { return int((i + (wordSize - 1)) >> log2WordSize) } // wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. // This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing bitmap). func wordsNeededUnbound(i uint) int { return int((i + (wordSize - 1)) >> log2WordSize) } // wordsIndex calculates the index of words in a `uint64` func wordsIndex(i uint) uint { return i & (wordSize - 1) } // New creates a new BitSet with a hint that length bits will be required func New(length uint) (bset *BitSet) { defer func() { Loading Loading @@ -135,9 +156,8 @@ func (b *BitSet) Len() uint { return b.length } // extendSetMaybe adds additional words to incorporate new bits if needed func (b *BitSet) extendSetMaybe(i uint) { if i >= b.length { // if we need more bits, make 'em // extendSet adds additional words to incorporate new bits if needed func (b *BitSet) extendSet(i uint) { if i >= Cap() { panic("You are exceeding the capacity") } Loading @@ -153,14 +173,13 @@ func (b *BitSet) extendSetMaybe(i uint) { } b.length = i + 1 } } // Test whether bit i is set. func (b *BitSet) Test(i uint) bool { if i >= b.length { return false } return b.set[i>>log2WordSize]&(1<<(i&(wordSize-1))) != 0 return b.set[i>>log2WordSize]&(1<<wordsIndex(i)) != 0 } // Set bit i to 1, the capacity of the bitset is automatically Loading @@ -170,8 +189,10 @@ func (b *BitSet) Test(i uint) bool { // may lead to a memory shortage and a panic: the caller is responsible // for providing sensible parameters in line with their memory capacity. func (b *BitSet) Set(i uint) *BitSet { b.extendSetMaybe(i) b.set[i>>log2WordSize] |= 1 << (i & (wordSize - 1)) if i >= b.length { // if we need more bits, make 'em b.extendSet(i) } b.set[i>>log2WordSize] |= 1 << wordsIndex(i) return b } Loading @@ -180,7 +201,7 @@ func (b *BitSet) Clear(i uint) *BitSet { if i >= b.length { return b } b.set[i>>log2WordSize] &^= 1 << (i & (wordSize - 1)) b.set[i>>log2WordSize] &^= 1 << wordsIndex(i) return b } Loading @@ -205,7 +226,7 @@ func (b *BitSet) Flip(i uint) *BitSet { if i >= b.length { return b.Set(i) } b.set[i>>log2WordSize] ^= 1 << (i & (wordSize - 1)) b.set[i>>log2WordSize] ^= 1 << wordsIndex(i) return b } Loading @@ -218,15 +239,18 @@ func (b *BitSet) FlipRange(start, end uint) *BitSet { if start >= end { return b } b.extendSetMaybe(end - 1) if end-1 >= b.length { // if we need more bits, make 'em b.extendSet(end - 1) } var startWord uint = start >> log2WordSize var endWord uint = end >> log2WordSize b.set[startWord] ^= ^(^uint64(0) << (start & (wordSize - 1))) b.set[startWord] ^= ^(^uint64(0) << wordsIndex(start)) for i := startWord; i < endWord; i++ { b.set[i] = ^b.set[i] } b.set[endWord] ^= ^uint64(0) >> (-end & (wordSize - 1)) if end&(wordSize-1) != 0 { b.set[endWord] ^= ^uint64(0) >> wordsIndex(-end) } return b } Loading Loading @@ -254,7 +278,10 @@ func (b *BitSet) Shrink(lastbitindex uint) *BitSet { copy(shrunk, b.set[:idx]) b.set = shrunk b.length = length b.set[idx-1] &= (allBits >> (uint64(64) - uint64(length&(wordSize-1)))) lastWordUsedBits := length % 64 if lastWordUsedBits != 0 { b.set[idx-1] &= allBits >> uint64(64-wordsIndex(lastWordUsedBits)) } return b } Loading Loading @@ -283,7 +310,7 @@ func (b *BitSet) Compact() *BitSet { // this method could be extremely slow and in some cases might cause the entire BitSet // to be recopied. func (b *BitSet) InsertAt(idx uint) *BitSet { insertAtElement := (idx >> log2WordSize) insertAtElement := idx >> log2WordSize // if length of set is a multiple of wordSize we need to allocate more space first if b.isLenExactMultiple() { Loading @@ -302,13 +329,13 @@ func (b *BitSet) InsertAt(idx uint) *BitSet { // generate a mask to extract the data that we need to shift left // within the element where we insert a bit dataMask := ^(uint64(1)<<uint64(idx&(wordSize-1)) - 1) dataMask := uint64(1)<<uint64(wordsIndex(idx)) - 1 // extract that data that we'll shift data := b.set[i] & dataMask data := b.set[i] & (^dataMask) // set the positions of the data mask to 0 in the element where we insert b.set[i] &= ^dataMask b.set[i] &= dataMask // shift data mask to the left and insert its data to the slice element b.set[i] |= data << 1 Loading Loading @@ -356,7 +383,7 @@ func (b *BitSet) DeleteAt(i uint) *BitSet { // generate a mask for the data that needs to be shifted right // within that slice element that gets modified dataMask := ^((uint64(1) << (i & (wordSize - 1))) - 1) dataMask := ^((uint64(1) << wordsIndex(i)) - 1) // extract the data that we'll shift right from the slice element data := b.set[deleteAtElement] & dataMask Loading Loading @@ -394,7 +421,7 @@ func (b *BitSet) NextSet(i uint) (uint, bool) { return 0, false } w := b.set[x] w = w >> (i & (wordSize - 1)) w = w >> wordsIndex(i) if w != 0 { return i + trailingZeroes64(w), true } Loading Loading @@ -423,7 +450,6 @@ func (b *BitSet) NextSet(i uint) (uint, bool) { // j += 1 // } // // // It is possible to retrieve all set bits as follow: // // indices := make([]uint, bitmap.Count()) Loading @@ -438,7 +464,7 @@ func (b *BitSet) NextSetMany(i uint, buffer []uint) (uint, []uint) { if x >= len(b.set) || capacity == 0 { return 0, myanswer[:0] } skip := i & (wordSize - 1) skip := wordsIndex(i) word := b.set[x] >> skip myanswer = myanswer[:capacity] size := int(0) Loading Loading @@ -481,8 +507,8 @@ func (b *BitSet) NextClear(i uint) (uint, bool) { return 0, false } w := b.set[x] w = w >> (i & (wordSize - 1)) wA := allBits >> (i & (wordSize - 1)) w = w >> wordsIndex(i) wA := allBits >> wordsIndex(i) index := i + trailingZeroes64(^w) if w != wA && index < b.length { return index, true Loading Loading @@ -510,7 +536,7 @@ func (b *BitSet) ClearAll() *BitSet { // wordCount returns the number of words used in a bit set func (b *BitSet) wordCount() int { return len(b.set) return wordsNeededUnbound(b.length) } // Clone this BitSet Loading @@ -522,9 +548,10 @@ func (b *BitSet) Clone() *BitSet { return c } // Copy into a destination BitSet // Returning the size of the destination BitSet // like array copy // Copy into a destination BitSet using the Go array copy semantics: // the number of bits copied is the minimum of the number of bits in the current // BitSet (Len()) and the destination Bitset. // We return the number of bits copied in the destination BitSet. func (b *BitSet) Copy(c *BitSet) (count uint) { if c == nil { return Loading @@ -536,8 +563,32 @@ func (b *BitSet) Copy(c *BitSet) (count uint) { if b.length < c.length { count = b.length } // Cleaning the last word is needed to keep the invariant that other functions, such as Count, require // that any bits in the last word that would exceed the length of the bitmask are set to 0. c.cleanLastWord() return } // CopyFull copies into a destination BitSet such that the destination is // identical to the source after the operation, allocating memory if necessary. func (b *BitSet) CopyFull(c *BitSet) { if c == nil { return } c.length = b.length if len(b.set) == 0 { if c.set != nil { c.set = c.set[:0] } } else { if cap(c.set) < len(b.set) { c.set = make([]uint64, len(b.set)) } else { c.set = c.set[:len(b.set)] } copy(c.set, b.set) } } // Count (number of set bits). // Also known as "popcount" or "population count". Loading @@ -561,10 +612,9 @@ func (b *BitSet) Equal(c *BitSet) bool { if b.length == 0 { // if they have both length == 0, then could have nil set return true } // testing for equality shoud not transform the bitset (no call to safeSet) for p, v := range b.set { if c.set[p] != v { wn := b.wordCount() for p:= 0; p < wn; p++ { if c.set[p] != b.set[p] { return false } } Loading Loading @@ -671,7 +721,9 @@ func (b *BitSet) InPlaceIntersection(compare *BitSet) { b.set[i] = 0 } if compare.length > 0 { b.extendSetMaybe(compare.length - 1) if compare.length-1 >= b.length { b.extendSet(compare.length - 1) } } } Loading Loading @@ -710,8 +762,8 @@ func (b *BitSet) InPlaceUnion(compare *BitSet) { if l > int(b.wordCount()) { l = int(b.wordCount()) } if compare.length > 0 { b.extendSetMaybe(compare.length - 1) if compare.length > 0 && compare.length-1 >= b.length { b.extendSet(compare.length - 1) } for i := 0; i < l; i++ { b.set[i] |= compare.set[i] Loading Loading @@ -758,8 +810,8 @@ func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { if l > int(b.wordCount()) { l = int(b.wordCount()) } if compare.length > 0 { b.extendSetMaybe(compare.length - 1) if compare.length > 0 && compare.length-1 >= b.length { b.extendSet(compare.length - 1) } for i := 0; i < l; i++ { b.set[i] ^= compare.set[i] Loading @@ -773,17 +825,17 @@ func (b *BitSet) InPlaceSymmetricDifference(compare *BitSet) { // Is the length an exact multiple of word sizes? func (b *BitSet) isLenExactMultiple() bool { return b.length%wordSize == 0 return wordsIndex(b.length) == 0 } // Clean last word by setting unused bits to 0 func (b *BitSet) cleanLastWord() { if !b.isLenExactMultiple() { b.set[len(b.set)-1] &= allBits >> (wordSize - b.length%wordSize) b.set[len(b.set)-1] &= allBits >> (wordSize - wordsIndex(b.length)) } } // Complement computes the (local) complement of a biset (up to length bits) // Complement computes the (local) complement of a bitset (up to length bits) func (b *BitSet) Complement() (result *BitSet) { panicIfNull(b) result = New(b.length) Loading Loading @@ -811,7 +863,6 @@ func (b *BitSet) None() bool { return false } } return true } return true } Loading Loading @@ -852,7 +903,8 @@ func (b *BitSet) DumpAsBits() string { // BinaryStorageSize returns the binary storage requirements func (b *BitSet) BinaryStorageSize() int { return binary.Size(uint64(0)) + binary.Size(b.set) nWords := b.wordCount() return binary.Size(uint64(0)) + binary.Size(b.set[:nWords]) } // WriteTo writes a BitSet to a stream Loading @@ -866,7 +918,19 @@ func (b *BitSet) WriteTo(stream io.Writer) (int64, error) { } // Write set err = binary.Write(stream, binaryOrder, b.set) // current implementation of bufio.Writer is more memory efficient than // binary.Write for large set writer := bufio.NewWriter(stream) var item = make([]byte, binary.Size(uint64(0))) // for serializing one uint64 nWords := b.wordCount() for i := range b.set[:nWords] { binaryOrder.PutUint64(item, b.set[i]) if nn, err := writer.Write(item); err != nil { return int64(i*binary.Size(uint64(0)) + nn), err } } err = writer.Flush() return int64(b.BinaryStorageSize()), err } Loading @@ -877,6 +941,9 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { // Read length first err := binary.Read(stream, binaryOrder, &length) if err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } return 0, err } newset := New(uint(length)) Loading @@ -885,11 +952,18 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { return 0, errors.New("unmarshalling error: type mismatch") } // Read remaining bytes as set err = binary.Read(stream, binaryOrder, newset.set) if err != nil { var item [8]byte nWords := wordsNeeded(uint(length)) reader := bufio.NewReader(io.LimitReader(stream, 8*int64(nWords))) for i := 0; i < nWords; i++ { if _, err := io.ReadFull(reader, item[:]); err != nil { if err == io.EOF { err = io.ErrUnexpectedEOF } return 0, err } newset.set[i] = binaryOrder.Uint64(item[:]) } *b = *newset return int64(b.BinaryStorageSize()), nil Loading @@ -898,25 +972,18 @@ func (b *BitSet) ReadFrom(stream io.Reader) (int64, error) { // MarshalBinary encodes a BitSet into a binary form and returns the result. func (b *BitSet) MarshalBinary() ([]byte, error) { var buf bytes.Buffer writer := bufio.NewWriter(&buf) _, err := b.WriteTo(writer) _, err := b.WriteTo(&buf) if err != nil { return []byte{}, err } err = writer.Flush() return buf.Bytes(), err } // UnmarshalBinary decodes the binary form generated by MarshalBinary. func (b *BitSet) UnmarshalBinary(data []byte) error { buf := bytes.NewReader(data) reader := bufio.NewReader(buf) _, err := b.ReadFrom(reader) _, err := b.ReadFrom(buf) return err } Loading