Слияние кода завершено, страница обновится автоматически
package riot
import (
"encoding/gob"
"fmt"
"log"
"os"
"testing"
"github.com/go-ego/riot/types"
"github.com/vcaesar/tt"
)
func TestEngineIndexWithNewStore(t *testing.T) {
gob.Register(ScoringFields{})
var engine = New("./testdata/test_dict.txt", "./riot.new", 8)
log.Println("new engine start...")
// engine = engine.New()
AddDocs(engine)
engine.RemoveDoc("5", true)
engine.Flush()
engine.Close()
// os.RemoveAll("riot.new")
// var engine1 = New("./testdata/test_dict.txt", "./riot.new")
var engine1 = New("./testdata/test_new.toml")
// engine1 = engine1.New()
log.Println("test...")
engine1.Flush()
log.Println("new engine1 start...")
outputs := engine1.Search(types.SearchReq{Text: reqText})
tt.Expect(t, "2", len(outputs.Tokens))
tt.Expect(t, "world", outputs.Tokens[0])
tt.Expect(t, "人口", outputs.Tokens[1])
outDocs := outputs.Docs.(types.ScoredDocs)
tt.Expect(t, "2", len(outDocs))
// tt.Expect(t, "2", outDocs[0].DocId)
tt.Expect(t, "2500", int(outDocs[0].Scores[0]*1000))
tt.Expect(t, "[]", outDocs[0].TokenSnippetLocs)
// tt.Expect(t, "1", outDocs[1].DocId)
tt.Expect(t, "2215", int(outDocs[1].Scores[0]*1000))
tt.Expect(t, "[]", outDocs[1].TokenSnippetLocs)
engine1.Close()
os.RemoveAll("riot.new")
// os.RemoveAll("riot-index")
}
var (
rankTestOpts = rankOptsMax(0, 1)
)
func testRankOpt(idOnly bool) types.EngineOpts {
return types.EngineOpts{
Using: 1,
IDOnly: idOnly,
GseDict: "./testdata/test_dict.txt",
DefRankOpts: &rankTestOpts,
IndexerOpts: inxOpts,
}
}
func lookupReq(engine *Engine) (types.SearchReq, []string, chan rankerReturnReq) {
request := types.SearchReq{
Text: reqText,
DocIds: makeDocIds(),
}
tokens := engine.Tokens(request)
// 建立排序器返回的通信通道
rankerReturnChan := make(
chan rankerReturnReq, engine.initOptions.NumShards)
// 生成查找请求
lookupRequest := indexerLookupReq{
countDocsOnly: request.CountDocsOnly,
tokens: tokens,
labels: request.Labels,
docIds: request.DocIds,
options: rankTestOpts,
rankerReturnChan: rankerReturnChan,
orderless: request.Orderless,
logic: request.Logic,
}
// 向索引器发送查找请求
for shard := 0; shard < engine.initOptions.NumShards; shard++ {
engine.indexerLookupChans[shard] <- lookupRequest
}
return request, tokens, rankerReturnChan
}
func TestDocRankID(t *testing.T) {
var engine Engine
engine.Init(testRankOpt(true))
AddDocs(&engine)
engine.RemoveDoc("5")
engine.Flush()
request, tokens, rankerReturnChan := lookupReq(&engine)
outputs := engine.RankID(request, rankTestOpts, tokens, rankerReturnChan)
if outputs.Docs != nil {
outDocs := outputs.Docs.(types.ScoredIDs)
tt.Expect(t, "1", len(outDocs))
}
tt.Expect(t, "2", len(outputs.Tokens))
tt.Expect(t, "2", outputs.NumDocs)
engine.Close()
}
func TestDocRanks(t *testing.T) {
var engine Engine
engine.Init(testRankOpt(false))
AddDocs(&engine)
engine.RemoveDoc("5")
engine.Flush()
request, tokens, rankerReturnChan := lookupReq(&engine)
outputs := engine.Ranks(request, rankTestOpts, tokens, rankerReturnChan)
if outputs.Docs != nil {
outDocs := outputs.Docs.(types.ScoredDocs)
tt.Expect(t, "1", len(outDocs))
}
tt.Expect(t, "2", len(outputs.Tokens))
tt.Expect(t, "2", outputs.NumDocs)
// test search
outputs1 := engine.Search(types.SearchReq{
Text: reqText,
Timeout: 1000,
DocIds: makeDocIds()})
if outputs1.Docs != nil {
outDocs1 := outputs.Docs.(types.ScoredDocs)
tt.Expect(t, "1", len(outDocs1))
}
tt.Expect(t, "2", len(outputs1.Tokens))
tt.Expect(t, "2", outputs1.NumDocs)
engine.Close()
}
func TestDocGetAllDocAndID(t *testing.T) {
gob.Register(ScoringFields{})
var engine Engine
opts := types.EngineOpts{
Using: 1,
NumShards: 5,
UseStore: true,
// StoreEngine: "bg",
StoreFolder: "riot.id",
IDOnly: true,
GseDict: "./testdata/test_dict.txt",
DefRankOpts: &rankTestOpts,
IndexerOpts: inxOpts,
}
engine.Init(opts)
AddDocs(&engine)
engine.RemoveDoc("5")
engine.Flush()
allIds := engine.GetDBAllIds()
fmt.Println("all id", allIds)
tt.Expect(t, "5", len(allIds))
tt.Expect(t, "[3 4 1 6 2]", allIds)
allIds = engine.GetAllDocIds()
fmt.Println("all doc id", allIds)
tt.Expect(t, "5", len(allIds))
tt.Expect(t, "[3 4 1 6 2]", allIds)
ids, docs := engine.GetDBAllDocs()
fmt.Println("all id and doc", allIds, docs)
tt.Expect(t, "5", len(ids))
tt.Expect(t, "5", len(docs))
tt.Expect(t, "[3 4 1 6 2]", ids)
allDoc := `[{The world <nil> [] [] <nil>} {有人口 <nil> [] [] {2 3 1}} {The world, 有七十亿人口人口 <nil> [] [] {1 2 3}} {有七十亿人口 <nil> [] [] {2 3 3}} {The world, 人口 <nil> [] [] <nil>}]`
tt.Expect(t, allDoc, docs)
has := engine.HasDoc("5")
tt.Expect(t, "false", has)
has = engine.HasDoc("2")
tt.Equal(t, true, has)
has = engine.HasDoc("3")
tt.Equal(t, true, has)
has = engine.HasDoc("4")
tt.Expect(t, "true", has)
dbhas := engine.HasDocDB("5")
tt.Expect(t, "false", dbhas)
dbhas = engine.HasDocDB("2")
tt.Equal(t, true, dbhas)
dbhas = engine.HasDocDB("3")
tt.Equal(t, true, dbhas)
dbhas = engine.HasDocDB("4")
tt.Expect(t, "true", dbhas)
docIds := make(map[string]bool)
docIds["5"] = true
docIds["1"] = true
outputs := engine.Search(types.SearchReq{
Text: reqText,
DocIds: docIds})
if outputs.Docs != nil {
outDocs := outputs.Docs.(types.ScoredIDs)
fmt.Println("output docs: ", outputs)
tt.Expect(t, "1", len(outDocs))
}
tt.Expect(t, "2", len(outputs.Tokens))
tt.Expect(t, "1", outputs.NumDocs)
engine.Close()
os.RemoveAll("riot.id")
}
func testOpts(use int, store string, args ...bool) types.EngineOpts {
var pinyin bool
if len(args) > 0 {
pinyin = args[0]
}
return types.EngineOpts{
// Using: 1,
Using: use,
UseStore: true,
StoreFolder: store,
PinYin: pinyin,
IDOnly: true,
GseDict: "./testdata/test_dict.txt",
}
}
func TestDocPinYin(t *testing.T) {
var engine, pinyinOpt Engine
engine.Init(testOpts(0, "riot.py"))
pinyinOpt.Init(testOpts(0, "riot.py.opt", true))
// AddDocs(&engine)
// engine.RemoveDoc(5)
tokens := engine.PinYin(text2)
fmt.Println("tokens...", tokens)
tt.Expect(t, "46", len(tokens))
var tokenDatas []types.TokenData
// tokens := []string{"z", "zl"}
for i := 0; i < len(tokens); i++ {
tokenData := types.TokenData{Text: tokens[i]}
tokenDatas = append(tokenDatas, tokenData)
}
index1 := types.DocData{Tokens: tokenDatas, Fields: "在路上"}
index2 := types.DocData{Content: text2, Tokens: tokenDatas}
engine.Index("10", index1)
engine.Index("11", index2)
engine.Flush()
data := types.DocData{Content: text2}
pinyinOpt.Index("10", data)
pinyinOpt.Index("11", data)
pinyinOpt.Flush()
docIds := make(map[string]bool)
docIds["5"] = true
docIds["10"] = true
docIds["11"] = true
pyOutputs := pinyinOpt.SearchID(types.SearchReq{
Text: "zl",
DocIds: docIds,
})
tt.Expect(t, "2", len(pyOutputs.Docs))
tt.Expect(t, "1", len(pyOutputs.Tokens))
tt.Expect(t, "2", pyOutputs.NumDocs)
outputs := engine.Search(types.SearchReq{
Text: "zl",
DocIds: docIds,
})
fmt.Println("outputs", outputs.Docs)
if outputs.Docs != nil {
outDocs := outputs.Docs.(types.ScoredIDs)
tt.Expect(t, "2", len(outDocs))
// tt.Expect(t, "11", outDocs[0].DocId)
// tt.Expect(t, "10", outDocs[1].DocId)
}
tt.Expect(t, "1", len(outputs.Tokens))
tt.Expect(t, "2", outputs.NumDocs)
engine.Close()
pinyinOpt.Close()
os.RemoveAll("riot.py")
os.RemoveAll("riot.py.opt")
}
func TestForSplitData(t *testing.T) {
var engine Engine
engine.Init(testOpts(4, "riot.data"))
AddDocs(&engine)
engine.RemoveDoc("5")
engine.Flush()
tokenDatas := engine.PinYin(text2)
tokens, num := engine.ForSplitData(tokenDatas, len(tokenDatas))
tt.Expect(t, "81", len(tokens))
tt.Expect(t, "92", num)
index1 := types.DocData{Content: "在路上"}
engine.Index("10", index1, true)
docIds := make(map[string]bool)
docIds["5"] = true
docIds["1"] = true
outputs := engine.Search(types.SearchReq{
Text: reqText,
DocIds: docIds})
if outputs.Docs != nil {
outDocs := outputs.Docs.(types.ScoredIDs)
tt.Expect(t, "0", len(outDocs))
}
tt.Expect(t, "2", len(outputs.Tokens))
tt.Expect(t, "0", outputs.NumDocs)
engine.Close()
os.RemoveAll("riot.data")
}
func testNum(t *testing.T, numAdd, numInx, numRm uint64) {
tt.Expect(t, "26", numAdd)
tt.Expect(t, "6", numInx)
tt.Expect(t, "8", numRm)
}
func TestDocCounters(t *testing.T) {
var engine Engine
engine.Init(testOpts(1, "riot.doc"))
AddDocs(&engine)
engine.RemoveDoc("5")
engine.Flush()
numAdd := engine.NumTokenAdded()
numInx := engine.NumIndexed()
numRm := engine.NumRemoved()
testNum(t, numAdd, numInx, numRm)
numAdd = engine.NumTokenIndexAdded()
numInx = engine.NumDocsIndexed()
numRm = engine.NumDocsRemoved()
testNum(t, numAdd, numInx, numRm)
docIds := make(map[string]bool)
docIds["5"] = true
docIds["1"] = true
outputs := engine.Search(types.SearchReq{
Text: reqText,
DocIds: docIds})
if outputs.Docs != nil {
outDocs := outputs.Docs.(types.ScoredIDs)
tt.Expect(t, "1", len(outDocs))
}
tt.Expect(t, "2", len(outputs.Tokens))
tt.Expect(t, "1", outputs.NumDocs)
engine.Close()
os.RemoveAll("riot.doc")
}
Вы можете оставить комментарий после Вход в систему
Неприемлемый контент может быть отображен здесь и не будет показан на странице. Вы можете проверить и изменить его с помощью соответствующей функции редактирования.
Если вы подтверждаете, что содержание не содержит непристойной лексики/перенаправления на рекламу/насилия/вульгарной порнографии/нарушений/пиратства/ложного/незначительного или незаконного контента, связанного с национальными законами и предписаниями, вы можете нажать «Отправить» для подачи апелляции, и мы обработаем ее как можно скорее.
Опубликовать ( 0 )