提问：70M左右的网页数据，索引出的文件在1.6G是否正常？ about gojieba HOT 6 CLOSED

  899.3 MiB [##########] /gojieba.bleve                                                                                                               
   43.3 MiB [          ] /source

package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/blevesearch/bleve"
	. "github.com/qiukeren/go-utils/common"
	"github.com/yanyiwu/gojieba"
	_ "github.com/yanyiwu/gojieba/bleve"
)

func Example() {
	INDEX_DIR := "gojieba.bleve"
	dirEntries, err := ioutil.ReadDir("/Users/XXX/gopath/src/search/bleve/source")
	if err != nil {
		log.Panicln(err)
	}
	type Message struct {
		Id      string
		Content string
	}

	indexMapping := bleve.NewIndexMapping()
	os.RemoveAll(INDEX_DIR)
	// clean index when example finished
	// defer os.RemoveAll(INDEX_DIR)

	err = indexMapping.AddCustomTokenizer("gojieba",
		map[string]interface{}{
			"dictpath":     gojieba.DICT_PATH,
			"hmmpath":      gojieba.HMM_PATH,
			"userdictpath": gojieba.USER_DICT_PATH,
			"type":         "gojieba",
			"idf":          gojieba.IDF_PATH, //idf 与stop_words必须要加，不然报错，此处采用自带的idf
			"stop_words":   gojieba.STOP_WORDS_PATH,
		},
	)
	if err != nil {
		panic(err)
	}
	err = indexMapping.AddCustomAnalyzer("gojieba",
		map[string]interface{}{
			"type":      "gojieba",
			"tokenizer": "gojieba",
		},
	)
	if err != nil {
		panic(err)
	}
	indexMapping.DefaultAnalyzer = "gojieba"

	index, err := bleve.New(INDEX_DIR, indexMapping)
	if err != nil {
		panic(err)
	}
	log.Println(len(dirEntries))
	for k, v := range dirEntries {
		log.Println(k, "of", len(dirEntries))

		data, _ := ReadToString("/Users/XXX/gopath/src/search/bleve/source/" + v.Name())
		message := Message{Id: v.Name(), Content: string(data)}

		// go func(name string, content Message) {
		index.Index(v.Name(), message)
		// }(v.Name(), message)

	}

	querys := []string{
		"你好世界",
		"亲口交代",
	}

	for _, q := range querys {
		req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(q))
		req.Highlight = bleve.NewHighlight()
		res, err := index.Search(req)
		if err != nil {
			panic(err)
		}
		fmt.Println(res)
	}
}

func main() {
	Example()
}