Update: update bleve and fixed #6, add newDoc function

.github/workflows/go.yml

@@ -10,10 +10,10 @@ jobs:
     runs-on: ${{ matrix.os }}
 
     steps:
-      - name: Set up Go 1.18
+      - name: Set up Go 1.21.0
         uses: actions/setup-go@v1
         with:
-          go-version: 1.18
+          go-version: 1.21.0
         id: go
 
       - name: Check out code into the Go module directory

analyzer.go

@@ -11,7 +11,7 @@ import (
 	"github.com/blevesearch/bleve/v2/registry"
 )
 
-func analyzerConstructor(config map[string]interface{}, cache *registry.Cache) (*analysis.Analyzer, error) {
+func NewAnalyzer(config map[string]interface{}, cache *registry.Cache) (analysis.Analyzer, error) {
 	tokenizerName, ok := config["tokenizer"].(string)
 	if !ok {
 		return nil, errors.New("must have tokenizer")
@@ -22,11 +22,11 @@ func analyzerConstructor(config map[string]interface{}, cache *registry.Cache) (
 		return nil, err
 	}
 
-	az := &analysis.Analyzer{Tokenizer: tokenizer}
+	az := &analysis.DefaultAnalyzer{Tokenizer: tokenizer}
 	return az, nil
 }
 
 func init() {
-	registry.RegisterAnalyzer(TokenName, analyzerConstructor)
-	registry.RegisterAnalyzer(SeparateName, analyzerConstructor)
+	registry.RegisterAnalyzer(TokenName, NewAnalyzer)
+	registry.RegisterAnalyzer(SeparateName, NewAnalyzer)
 }

bleve.go

@@ -142,13 +142,14 @@ func (c *GseCut) Tokenize(text []byte) analysis.TokenStream {
 	result := make(analysis.TokenStream, 0)
 	t1 := string(text)
 	cuts := c.Trim(c.Cut(t1, c.opt))
-	// fmt.Println("cuts: ", cuts)
+
 	azs := c.seg.Analyze(cuts, t1)
 	for _, az := range azs {
 		token := analysis.Token{
-			Term:     []byte(az.Text),
-			Start:    az.Start,
-			End:      az.End,
+			Term:  []byte(az.Text),
+			Start: az.Start,
+			End:   az.End,
+
 			Position: az.Position,
 			Type:     analysis.Ideographic,
 		}
@@ -162,6 +163,7 @@ func (s *Separator) Tokenize(text []byte) analysis.TokenStream {
 	result := make(analysis.TokenStream, 0)
 	t1 := string(text)
 	cuts := s.Trim(strings.Split(t1, s.sep))
+
 	azs := s.seg.Analyze(cuts, t1)
 	for _, az := range azs {
 		token := analysis.Token{

circle.yml

@@ -3,7 +3,7 @@ version: 2
 jobs:
   build:
     docker:
-      - image: golang:1.18.2
+      - image: golang:1.21.0
     working_directory: /gopath/src/github.com/vcaesar/gse-bleve
     steps:
       - checkout

go.mod

@@ -3,32 +3,38 @@ module github.com/vcaesar/gse-bleve
 go 1.17
 
 require (
-	github.com/blevesearch/bleve/v2 v2.3.2
+	github.com/blevesearch/bleve/v2 v2.3.9
 	github.com/go-ego/gse v0.70.2
 	github.com/vcaesar/tt v0.20.0
 )
 
 require (
-	github.com/RoaringBitmap/roaring v0.9.4 // indirect
-	github.com/bits-and-blooms/bitset v1.2.0 // indirect
-	github.com/blevesearch/bleve_index_api v1.0.1 // indirect
+	github.com/RoaringBitmap/roaring v1.2.3 // indirect
+	github.com/bits-and-blooms/bitset v1.4.0 // indirect
+	github.com/blevesearch/bleve_index_api v1.0.5 // indirect
+	github.com/blevesearch/geo v0.1.17 // indirect
 	github.com/blevesearch/go-porterstemmer v1.0.3 // indirect
 	github.com/blevesearch/gtreap v0.1.1 // indirect
-	github.com/blevesearch/mmap-go v1.0.3 // indirect
-	github.com/blevesearch/scorch_segment_api/v2 v2.1.0 // indirect
-	github.com/blevesearch/segment v0.9.0 // indirect
+	github.com/blevesearch/mmap-go v1.0.4 // indirect
+	github.com/blevesearch/scorch_segment_api/v2 v2.1.5 // indirect
+	github.com/blevesearch/segment v0.9.1 // indirect
 	github.com/blevesearch/snowballstem v0.9.0 // indirect
-	github.com/blevesearch/upsidedown_store_api v1.0.1 // indirect
-	github.com/blevesearch/vellum v1.0.7 // indirect
-	github.com/blevesearch/zapx/v11 v11.3.3 // indirect
-	github.com/blevesearch/zapx/v12 v12.3.3 // indirect
-	github.com/blevesearch/zapx/v13 v13.3.3 // indirect
-	github.com/blevesearch/zapx/v14 v14.3.3 // indirect
-	github.com/blevesearch/zapx/v15 v15.3.3 // indirect
-	github.com/golang/protobuf v1.3.2 // indirect
-	github.com/golang/snappy v0.0.2-0.20190904063534-ff6b7dc882cf // indirect
+	github.com/blevesearch/upsidedown_store_api v1.0.2 // indirect
+	github.com/blevesearch/vellum v1.0.10 // indirect
+	github.com/blevesearch/zapx/v11 v11.3.9 // indirect
+	github.com/blevesearch/zapx/v12 v12.3.9 // indirect
+	github.com/blevesearch/zapx/v13 v13.3.9 // indirect
+	github.com/blevesearch/zapx/v14 v14.3.9 // indirect
+	github.com/blevesearch/zapx/v15 v15.3.12 // indirect
+	github.com/golang/geo v0.0.0-20210211234256-740aa86cb551 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/snappy v0.0.4 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/mschoch/smat v0.2.0 // indirect
 	github.com/vcaesar/cedar v0.20.1 // indirect
-	go.etcd.io/bbolt v1.3.5 // indirect
-	golang.org/x/sys v0.0.0-20220429121018-84afa8d3f7b3 // indirect
+	go.etcd.io/bbolt v1.3.7 // indirect
+	golang.org/x/sys v0.4.0 // indirect
+	google.golang.org/protobuf v1.28.1 // indirect
 )