Skip to content
This repository has been archived by the owner on Oct 13, 2021. It is now read-only.

use leveledb to store,but save failed #84

Open
busyfree opened this issue Mar 5, 2019 · 2 comments
Open

use leveledb to store,but save failed #84

busyfree opened this issue Mar 5, 2019 · 2 comments

Comments

@busyfree
Copy link

busyfree commented Mar 5, 2019

  • Ego version (or commit ref):
  • Go version:go version go1.11 linux/amd64
  • Gcc version: 4.9.4 (GCC)
  • Operating system and bit:CentOS release 6.8 (Final) 64bit
  • Provide example code:
    beego main.go init server

func main() {
	if beego.BConfig.RunMode == "dev" {
		beego.BConfig.WebConfig.DirectoryIndex = true
		beego.BConfig.WebConfig.StaticDir["/swagger"] = "swagger"
	}
	initSearcher()
	beego.BeeApp.Server.RegisterOnShutdown(
		func() {
			initClose()
		})
	beego.Run()
}

func initSearcher() {
	// 初始化
	var dictFile = filepath.Join(beego.AppPath, "conf/dictionary.txt")
	var stopTokenFile = filepath.Join(beego.AppPath, "conf/stop_tokens.txt")
	controllers.Searcher.Init(types.EngineOpts{
		Using:         1,
		GseDict:       dictFile,
		StopTokenFile: stopTokenFile,
		IndexerOpts: &types.IndexerOpts{
			IndexType: types.LocsIndex,
		},
		UseStore:    true,
		StoreOnly:   true,
		StoreFolder: "/tmp/shareyou_search",
		StoreEngine: "ldb",
	})

}
func initClose() {
	controllers.Searcher.Flush()
	controllers.Searcher.Close()
}

api post to save data into engine
api.go

package controllers
import (
	"fmt"
	"github.com/go-ego/riot/types"
	"lib/utils"
	"reflect"
	"strings"
)

var (
	Searcher         = riot.Engine{}
)

const (
	// SecondsInADay seconds in a day
	SecondsInADay = 86400
	// MaxTokenProximity max token proximity
	MaxTokenProximity = 2
)

type ApiController struct {
	BaseController
}

type DocItem struct {
	Id        string `json:"id"`
	Timestamp uint64 `json:"timestamp"`
	UserName  string `json:"username"`
	LikeCount uint64 `json:"count"`
	Text      string `json:"text"`
	DocId     string `json:"docid"`
}

// WeiboScoringFields  weibo scoring fields
type DocItemScoringFields struct {
	Id        int64
	Type      string
	Timestamp int64
	LikeCount int64
}

// WeiboScoringCriteria custom weibo scoring criteria
type WeiboScoringCriteria struct {
}

// Score score and sort
func (criteria WeiboScoringCriteria) Score(doc types.IndexedDoc, fields interface{}) []float32 {
	if reflect.TypeOf(fields) != reflect.TypeOf(DocItemScoringFields{}) {
		return []float32{}
	}
	wsf := fields.(DocItemScoringFields)
	output := make([]float32, 3)
	if doc.TokenProximity > MaxTokenProximity {
		output[0] = 1.0 / float32(doc.TokenProximity)
	} else {
		output[0] = 1.0
	}
	output[1] = float32(wsf.Timestamp / (SecondsInADay * 3))
	output[2] = float32(doc.BM25 * (1 + float32(wsf.LikeCount)/10000))
	return output
}


func (o *ApiController) Post() {

	content := o.Input().Get("content")
	typeStr := o.Input().Get("type")
	ts := utils.MsTimestampNow()
	id, _ := o.GetInt64("id", 0)
	count, _ := o.GetInt64("count", 1)
	sfId, err := SnowFlakeServer.NextID()
	docId := fmt.Sprintf("%d", sfId)
	Searcher.Index(docId, types.DocData{
		Content: content,
		Fields: DocItemScoringFields{
			Id:        id,
			Type:      typeStr,
			Timestamp: ts,
			LikeCount: count,
		},
	})
	Searcher.Flush()
	return
}

func (o *ApiController) Get() {
	keyword := o.Input().Get("keyword")
	output := Searcher.SearchDoc(types.SearchReq{
		Text: keyword,
		RankOpts: &types.RankOpts{
			ScoringCriteria: &WeiboScoringCriteria{},
			OutputOffset:    0,
			MaxOutputs:      100,
		},
	})

	docs := make([]*DocItem, 0, 0)
	for _, doc := range output.Docs {
		item := DocItem{}
		field, _ := doc.Fields.(DocItemScoringFields)
		item.DocId = doc.DocId
		item.Text = doc.Content
		item.Timestamp = uint64(field.Timestamp)
		for _, t := range output.Tokens {
			item.Text = strings.Replace(item.Text, t, "<font color=red>"+t+"</font>", -1)
		}
		docs = append(docs, &item)
	}
	return
}

Description

when ctrl+c to exit the server ,then run again, search the keyword ,but nothing find. any help?

@busyfree busyfree reopened this Mar 13, 2019
@busyfree
Copy link
Author

why leveldb opened at rito/engine.go ,but when write data into leveldb then return leveldb status is closed.
image

@busyfree busyfree changed the title use leveldb store to save the data,but not reindex to engine when restart the server? use leveledb to store,but save failed Mar 13, 2019
@busyfree
Copy link
Author

Add a retry label to control the error.

image

Sign up for free to subscribe to this conversation on GitHub. Already have an account? Sign in.
Labels
None yet
Projects
None yet
Development

No branches or pull requests

1 participant