diff --git a/Gopkg.lock b/Gopkg.lock index dede9ac3476915f653f8a21fdab014447224c4ea..661cbc8393628bf7f2b679e32d4f37a29bb50e63 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -31,7 +31,7 @@ revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] - digest = "1:109a9e34c738730a57b729573cceffd58d082250337e5d7185c9ff8de3162a85" + digest = "1:c4463bdc81ad05a0540a53e4a0e7895bb6f1c18870a919b7c5befdad26c18fe9" name = "github.com/blevesearch/bleve" packages = [ ".", @@ -74,7 +74,7 @@ "size", ] pruneopts = "" - revision = "89a815df0798b34feb44e4d3ade3c3a9d338aa63" + revision = "5cf4e4e02e8a1aec8df8c6773d08e364ee694cea" [[projects]] digest = "1:b9a6b4e079fea1e137c0f8caeb65030978a8dee9f37df5a6ffd9fabe98cdd32e" diff --git a/Gopkg.toml b/Gopkg.toml index 2d993110768adcb2e0a37d66d77f5473a7b03603..45dd80f5ee3423f82cba66d3524d0e48c7ad4a77 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -72,7 +72,7 @@ [[constraint]] name = "github.com/blevesearch/bleve" - revision = "89a815df0798b34feb44e4d3ade3c3a9d338aa63" + revision = "5cf4e4e02e8a1aec8df8c6773d08e364ee694cea" [[constraint]] name = "gopkg.in/natefinch/lumberjack.v2" diff --git a/vendor/github.com/blevesearch/bleve/index.go b/vendor/github.com/blevesearch/bleve/index.go index f9462a41da2fd599582f3dc8bb939e9a2ce32df1..99357eee01127e995af3d7a107f4f1379fe5efeb 100644 --- a/vendor/github.com/blevesearch/bleve/index.go +++ b/vendor/github.com/blevesearch/bleve/index.go @@ -129,6 +129,14 @@ func (b *Batch) Merge(o *Batch) { } } +func (b *Batch) SetPersistedCallback(f index.BatchCallback) { + b.internal.SetPersistedCallback(f) +} + +func (b *Batch) PersistedCallback() index.BatchCallback { + return b.internal.PersistedCallback() +} + // An Index implements all the indexing and searching // capabilities of bleve. An Index can be created // using the New() and Open() methods. diff --git a/vendor/github.com/blevesearch/bleve/index/index.go b/vendor/github.com/blevesearch/bleve/index/index.go index a44046134a6dabf2992720dc9196c6246fe4566e..4ae79056e18d09d94481c041f1d162505af11a25 100644 --- a/vendor/github.com/blevesearch/bleve/index/index.go +++ b/vendor/github.com/blevesearch/bleve/index/index.go @@ -248,9 +248,12 @@ type DocIDReader interface { Close() error } +type BatchCallback func(error) + type Batch struct { - IndexOps map[string]*document.Document - InternalOps map[string][]byte + IndexOps map[string]*document.Document + InternalOps map[string][]byte + persistedCallback BatchCallback } func NewBatch() *Batch { @@ -276,6 +279,14 @@ func (b *Batch) DeleteInternal(key []byte) { b.InternalOps[string(key)] = nil } +func (b *Batch) SetPersistedCallback(f BatchCallback) { + b.persistedCallback = f +} + +func (b *Batch) PersistedCallback() BatchCallback { + return b.persistedCallback +} + func (b *Batch) String() string { rv := fmt.Sprintf("Batch (%d ops, %d internal ops)\n", len(b.IndexOps), len(b.InternalOps)) for k, v := range b.IndexOps { @@ -298,6 +309,7 @@ func (b *Batch) String() string { func (b *Batch) Reset() { b.IndexOps = make(map[string]*document.Document) b.InternalOps = make(map[string][]byte) + b.persistedCallback = nil } func (b *Batch) Merge(o *Batch) { diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go b/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go index 12f27af66ca2ce8e6dadb26c6a7da3cd7d192c57..ac627796f5c0788e76ddc5d47992e48dd9aea2a4 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go +++ b/vendor/github.com/blevesearch/bleve/index/scorch/introducer.go @@ -19,6 +19,7 @@ import ( "sync/atomic" "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/scorch/segment" "github.com/blevesearch/bleve/index/scorch/segment/zap" ) @@ -30,8 +31,9 @@ type segmentIntroduction struct { ids []string internal map[string][]byte - applied chan error - persisted chan error + applied chan error + persisted chan error + persistedCallback index.BatchCallback } type persistIntroduction struct { @@ -213,6 +215,9 @@ func (s *Scorch) introduceSegment(next *segmentIntroduction) error { if next.persisted != nil { s.rootPersisted = append(s.rootPersisted, next.persisted) } + if next.persistedCallback != nil { + s.persistedCallbacks = append(s.persistedCallbacks, next.persistedCallback) + } // swap in new index snapshot newSnapshot.epoch = s.nextSnapshotEpoch s.nextSnapshotEpoch++ diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go b/vendor/github.com/blevesearch/bleve/index/scorch/persister.go index 8d54b3a703174ecc6f77c2a3976a0f3202b37d44..2ba50867da747047ccdc5c2b24ef829619b00bbe 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/persister.go +++ b/vendor/github.com/blevesearch/bleve/index/scorch/persister.go @@ -30,6 +30,7 @@ import ( "time" "github.com/RoaringBitmap/roaring" + "github.com/blevesearch/bleve/index" "github.com/blevesearch/bleve/index/scorch/segment" "github.com/blevesearch/bleve/index/scorch/segment/zap" "github.com/boltdb/bolt" @@ -115,6 +116,7 @@ OUTER: var ourSnapshot *IndexSnapshot var ourPersisted []chan error + var ourPersistedCallbacks []index.BatchCallback // check to see if there is a new snapshot to persist s.rootLock.Lock() @@ -123,6 +125,8 @@ OUTER: ourSnapshot.AddRef() ourPersisted = s.rootPersisted s.rootPersisted = nil + ourPersistedCallbacks = s.persistedCallbacks + s.persistedCallbacks = nil atomic.StoreUint64(&s.iStats.persistSnapshotSize, uint64(ourSnapshot.Size())) atomic.StoreUint64(&s.iStats.persistEpoch, ourSnapshot.epoch) } @@ -150,6 +154,9 @@ OUTER: atomic.AddUint64(&s.stats.TotPersistLoopErr, 1) continue OUTER } + for i := range ourPersistedCallbacks { + ourPersistedCallbacks[i](err) + } atomic.StoreUint64(&s.stats.LastPersistedEpoch, ourSnapshot.epoch) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go b/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go index 5e56c49b03d45f2238c39a030c1617d66cf01ed2..07568e6e5a7c134c9c47ec24c94a10cf4697c111 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go +++ b/vendor/github.com/blevesearch/bleve/index/scorch/scorch.go @@ -54,6 +54,7 @@ type Scorch struct { rootLock sync.RWMutex root *IndexSnapshot // holds 1 ref-count on the root rootPersisted []chan error // closed when root is persisted + persistedCallbacks []index.BatchCallback nextSnapshotEpoch uint64 eligibleForRemoval []uint64 // Index snapshot epochs that are safe to GC. ineligibleForRemoval map[string]bool // Filenames that should not be GC'ed yet. @@ -355,7 +356,7 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { atomic.AddUint64(&s.stats.TotBatchesEmpty, 1) } - err = s.prepareSegment(newSegment, ids, batch.InternalOps) + err = s.prepareSegment(newSegment, ids, batch.InternalOps, batch.PersistedCallback()) if err != nil { if newSegment != nil { _ = newSegment.Close() @@ -375,16 +376,17 @@ func (s *Scorch) Batch(batch *index.Batch) (err error) { } func (s *Scorch) prepareSegment(newSegment segment.Segment, ids []string, - internalOps map[string][]byte) error { + internalOps map[string][]byte, persistedCallback index.BatchCallback) error { // new introduction introduction := &segmentIntroduction{ - id: atomic.AddUint64(&s.nextSegmentID, 1), - data: newSegment, - ids: ids, - obsoletes: make(map[uint64]*roaring.Bitmap), - internal: internalOps, - applied: make(chan error), + id: atomic.AddUint64(&s.nextSegmentID, 1), + data: newSegment, + ids: ids, + obsoletes: make(map[uint64]*roaring.Bitmap), + internal: internalOps, + applied: make(chan error), + persistedCallback: persistedCallback, } if !s.unsafeBatch { @@ -524,7 +526,7 @@ func (s *Scorch) Analyze(d *document.Document) *index.AnalysisResult { rv.Analyzed[i] = tokenFreqs rv.Length[i] = fieldLength - if len(d.CompositeFields) > 0 { + if len(d.CompositeFields) > 0 && field.Name() != "_id" { // see if any of the composite fields need this for _, compositeField := range d.CompositeFields { compositeField.Compose(field.Name(), fieldLength, tokenFreqs) diff --git a/vendor/github.com/blevesearch/bleve/index/scorch/scorch_test.go b/vendor/github.com/blevesearch/bleve/index/scorch/scorch_test.go index 424afbfc6c7c5ba3df525601e6c8dd370fd8a7aa..a1691fd09f1ca248aaa21f135b5a519b06231055 100644 --- a/vendor/github.com/blevesearch/bleve/index/scorch/scorch_test.go +++ b/vendor/github.com/blevesearch/bleve/index/scorch/scorch_test.go @@ -15,6 +15,7 @@ package scorch import ( + "fmt" "log" "os" "reflect" @@ -930,6 +931,65 @@ func TestIndexBatch(t *testing.T) { } } +func TestIndexBatchWithCallbacks(t *testing.T) { + cfg := CreateConfig("TestIndexBatchWithCallbacks") + defer func() { + err := DestroyTest(cfg) + if err != nil { + t.Fatal(err) + } + }() + + analysisQueue := index.NewAnalysisQueue(1) + idx, err := NewScorch(Name, cfg, analysisQueue) + if err != nil { + t.Fatal(err) + } + err = idx.Open() + if err != nil { + t.Fatalf("error opening index: %v", err) + } + defer func() { + err := idx.Close() + if err != nil { + t.Fatal(err) + } + }() + + // Check that callback function works + updated := false + cbErr := fmt.Errorf("") + + batch := index.NewBatch() + doc := document.NewDocument("3") + doc.AddField(document.NewTextField("name", []uint64{}, []byte("test3"))) + batch.Update(doc) + batch.SetPersistedCallback(func(e error) { + updated = true + cbErr = e + + }) + + err = idx.Batch(batch) + if err != nil { + t.Error(err) + } + + for i := 0; i < 30; i++ { + if updated { + break + } + time.Sleep(500 * time.Millisecond) + } + if !updated { + t.Fatal("Callback function wasn't called") + } + if cbErr != nil { + t.Fatal("Error wasn't updated properly on callback function") + } + +} + func TestIndexInsertUpdateDeleteWithMultipleTypesStored(t *testing.T) { cfg := CreateConfig("TestIndexInsertUpdateDeleteWithMultipleTypesStored") err := InitTest(cfg) diff --git a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go index 6d37385398997b94d98a62f1bb6720501cad5c96..8edbb5b3d62142309ab84b2d4a1423574577a613 100644 --- a/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go +++ b/vendor/github.com/blevesearch/bleve/index/upsidedown/upsidedown.go @@ -958,6 +958,11 @@ func (udc *UpsideDownCouch) Batch(batch *index.Batch) (err error) { } else { atomic.AddUint64(&udc.stats.errors, 1) } + + persistedCallback := batch.PersistedCallback() + if persistedCallback != nil { + persistedCallback(err) + } return } diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go b/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go index 9c59ceddeaad978f214520f8cc8aa577d8b31969..3a9cf23983ca229e24def2d134bc84e6589595c4 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string_lex.go @@ -273,6 +273,7 @@ func inNumOrStrState(l *queryStringLex, next rune, eof bool) (lexState, bool) { // see where to go if !l.seenDot && next == '.' { // stay in this state + l.seenDot = true l.buf += string(next) return inNumOrStrState, true } else if unicode.IsDigit(next) { diff --git a/vendor/github.com/blevesearch/bleve/search/query/query_string_parser_test.go b/vendor/github.com/blevesearch/bleve/search/query/query_string_parser_test.go index 356127fdc476b40c423b1c9f66ca78f2ec3a40a0..f10170edde5d00fc77c5876162f2a88ea3617cac 100644 --- a/vendor/github.com/blevesearch/bleve/search/query/query_string_parser_test.go +++ b/vendor/github.com/blevesearch/bleve/search/query/query_string_parser_test.go @@ -49,6 +49,16 @@ func TestQuerySyntaxParserValid(t *testing.T) { }, nil), }, + { + input: "127.0.0.1", + mapping: mapping.NewIndexMapping(), + result: NewBooleanQueryForQueryString( + nil, + []Query{ + NewMatchQuery("127.0.0.1"), + }, + nil), + }, { input: `"test phrase 1"`, mapping: mapping.NewIndexMapping(), diff --git a/vendor/github.com/blevesearch/bleve/test/versus_score_test.go b/vendor/github.com/blevesearch/bleve/test/versus_score_test.go new file mode 100644 index 0000000000000000000000000000000000000000..dcaf8d650485ab95fa6e5a486c7d104c46d0b312 --- /dev/null +++ b/vendor/github.com/blevesearch/bleve/test/versus_score_test.go @@ -0,0 +1,135 @@ +// Copyright (c) 2018 Couchbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package test + +import ( + "fmt" + "os" + "strconv" + "testing" + + "github.com/blevesearch/bleve" + "github.com/blevesearch/bleve/document" + "github.com/blevesearch/bleve/index/scorch" + "github.com/blevesearch/bleve/index/upsidedown" + "github.com/blevesearch/bleve/mapping" + "github.com/blevesearch/bleve/search" +) + +func TestDisjunctionSearchScoreIndexWithCompositeFields(t *testing.T) { + upHits := disjunctionQueryiOnIndexWithCompositeFields(upsidedown.Name, t) + scHits := disjunctionQueryiOnIndexWithCompositeFields(scorch.Name, t) + + if upHits[0].ID != scHits[0].ID || upHits[1].ID != scHits[1].ID { + t.Errorf("upsidedown, scorch returned different docs;\n"+ + "upsidedown: (%s, %s), scorch: (%s, %s)\n", + upHits[0].ID, upHits[1].ID, scHits[0].ID, scHits[1].ID) + } + + if scHits[0].Score != upHits[0].Score || scHits[1].Score != upHits[1].Score { + t.Errorf("upsidedown, scorch showing different scores;\n"+ + "upsidedown: (%+v, %+v), scorch: (%+v, %+v)\n", + *upHits[0].Expl, *upHits[1].Expl, *scHits[0].Expl, *scHits[1].Expl) + } + +} + +func disjunctionQueryiOnIndexWithCompositeFields(indexName string, + t *testing.T) []*search.DocumentMatch { + // create an index + idxMapping := mapping.NewIndexMapping() + idx, err := bleve.NewUsing("testidx", idxMapping, indexName, + bleve.Config.DefaultKVStore, nil) + if err != nil { + t.Error(err) + } + + defer func() { + err = idx.Close() + if err != nil { + t.Error(err) + } + err = os.RemoveAll("testidx") + if err != nil { + t.Error(err) + } + }() + + // create and insert documents as a batch + batch := idx.NewBatch() + docs := []struct { + field1 string + field2 int + }{ + { + field1: "one", + field2: 1, + }, + { + field1: "two", + field2: 2, + }, + } + + for i := 0; i < len(docs); i++ { + doc := document.NewDocument(strconv.Itoa(docs[i].field2)) + doc.Fields = []document.Field{ + document.NewTextField("field1", []uint64{}, []byte(docs[i].field1)), + document.NewNumericField("field2", []uint64{}, float64(docs[i].field2)), + } + doc.CompositeFields = []*document.CompositeField{ + document.NewCompositeFieldWithIndexingOptions( + "_all", true, []string{"field1"}, []string{}, + document.IndexField|document.IncludeTermVectors), + } + if err = batch.IndexAdvanced(doc); err != nil { + t.Error(err) + } + } + if err = idx.Batch(batch); err != nil { + t.Error(err) + } + + /* + Query: + DISJ + / \ + CONJ TERM(two) + / + TERM(one) + */ + + tq1 := bleve.NewTermQuery("one") + tq1.SetBoost(2) + tq2 := bleve.NewTermQuery("two") + tq2.SetBoost(3) + + cq := bleve.NewConjunctionQuery(tq1) + cq.SetBoost(4) + + q := bleve.NewDisjunctionQuery(tq1, tq2) + sr := bleve.NewSearchRequestOptions(q, 2, 0, true) + res, err := idx.Search(sr) + if err != nil { + t.Error(err) + } + + if len(res.Hits) != 2 { + t.Errorf(fmt.Sprintf("indexType: %s Expected 2 hits, "+ + "but got: %v", indexName, len(res.Hits))) + } + + return res.Hits +}