代码拉取完成,页面将自动刷新
// Copyright 2016 ego authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.
package gse
import (
"bufio"
"log"
"os"
"path"
"strings"
)
// StopWordMap the default stop words.
var StopWordMap = map[string]bool{
" ": true,
}
// LoadStopArr load stop word by []string
func (seg *Segmenter) LoadStopArr(dict []string) {
if seg.StopWordMap == nil {
seg.StopWordMap = make(map[string]bool)
}
for _, d := range dict {
seg.StopWordMap[d] = true
}
}
// LoadStop load stop word files add token to map
func (seg *Segmenter) LoadStop(files ...string) error {
if seg.StopWordMap == nil {
seg.StopWordMap = make(map[string]bool)
}
dictDir := path.Join(path.Dir(GetCurrentFilePath()), "data")
if len(files) <= 0 {
dictPath := path.Join(dictDir, "dict/zh/stop_word.txt")
files = append(files, dictPath)
}
name := strings.Split(files[0], ", ")
if name[0] == "zh" {
name[0] = path.Join(dictDir, "dict/zh/stop_tokens.txt")
}
for i := 0; i < len(name); i++ {
if !seg.SkipLog {
log.Printf("Load the stop word dictionary: \"%s\" ", name[i])
}
file, err := os.Open(name[i])
if err != nil {
log.Printf("Could not load dictionaries: \"%s\", %v \n", name[i], err)
return err
}
defer file.Close()
scanner := bufio.NewScanner(file)
for scanner.Scan() {
text := scanner.Text()
if text != "" {
seg.StopWordMap[text] = true
}
}
}
return nil
}
// AddStop add a token to the StopWord dictionary.
func (seg *Segmenter) AddStop(text string) {
seg.StopWordMap[text] = true
}
// RemoveStop remove a token from the StopWord dictionary.
func (seg *Segmenter) RemoveStop(text string) {
delete(seg.StopWordMap, text)
}
// EmptyStop empty the stop dictionary
func (seg *Segmenter) EmptyStop() error {
seg.StopWordMap = nil
return nil
}
// IsStop check the word is a stop word.
func (seg *Segmenter) IsStop(s string) bool {
_, ok := seg.StopWordMap[s]
return ok
// return StopWordMap[s]
}
此处可能存在不合适展示的内容,页面不予展示。您可通过相关编辑功能自查并修改。
如您确认内容无涉及 不当用语 / 纯广告导流 / 暴力 / 低俗色情 / 侵权 / 盗版 / 虚假 / 无价值内容或违法国家有关法律法规的内容,可点击提交进行申诉,我们将尽快为您处理。