adds stopwords detection to searcher
@@ -19,6 +19,7 @@ import (
"github.com/pkg/errors"
"github.com/semi-technologies/weaviate/adapters/repos/db/helpers"
+ "github.com/semi-technologies/weaviate/adapters/repos/db/inverted/stopwords"
"github.com/semi-technologies/weaviate/adapters/repos/db/lsmkv"
"github.com/semi-technologies/weaviate/adapters/repos/db/notimplemented"
"github.com/semi-technologies/weaviate/adapters/repos/db/propertyspecific"
@@ -238,8 +239,13 @@ func (fs *Searcher) extractPropValuePair(filter *filters.Clause,
return nil, err
}
+ sd, err := fs.stopwordDetector(className)
+ if err != nil {
+ return nil, err
+ }
+
return fs.extractTokenizableProp(props[0], filter.Value.Type, filter.Value.Value,
- filter.Operator, property.Tokenization)
+ filter.Operator, property.Tokenization, sd)
}
return fs.extractPrimitiveProp(props[0], filter.Value.Type, filter.Value.Value,
@@ -338,7 +344,7 @@ func (fs *Searcher) extractIDProp(value interface{},
}
func (fs *Searcher) extractTokenizableProp(propName string, dt schema.DataType, value interface{},
- operator filters.Operator, tokenization string) (*propValuePair, error) {
+ operator filters.Operator, tokenization string, sd stopwordDetector) (*propValuePair, error) {
var parts []string
switch dt {
@@ -368,21 +374,27 @@ func (fs *Searcher) extractTokenizableProp(propName string, dt schema.DataType,
return nil, fmt.Errorf("expected value type to be string or text, got %v", dt)
}
- propValuePairs := make([]*propValuePair, len(parts))
- for i, part := range parts {
- propValuePairs[i] = &propValuePair{
+ propValuePairs := make([]*propValuePair, 0, len(parts))
+ for _, part := range parts {
+ if sd.IsStopword(part) {
+ continue
+ }
+ propValuePairs = append(propValuePairs, &propValuePair{
value: []byte(part),
hasFrequency: true,
prop: propName,
operator: operator,
- }
+ })
}
- if len(parts) > 1 {
+ if len(propValuePairs) > 1 {
return &propValuePair{operator: filters.OperatorAnd, children: propValuePairs}, nil
}
+ if len(propValuePairs) == 1 {
return propValuePairs[0], nil
}
+ return nil, errors.Errorf("invalid search term, only stopwords provided. Stopwords can be configured in class.invertedIndexConfig.stopwords")
+}
// TODO: repeated calls to on... aren't too efficient because we iterate over
// the schema each time, might be smarter to have a single method that
@@ -423,6 +435,26 @@ func (fs *Searcher) onTokenizablePropValue(valueType schema.DataType) bool {
}
}
+func (fs *Searcher) stopwordDetector(className schema.ClassName) (stopwordDetector, error) {
+ class := fs.schema.GetClass(className)
+ if class == nil {
+ return nil, errors.Errorf("Unknown Class '%v'", className)
+ }
+
+ iicm := &models.InvertedIndexConfig{}
+ if class.InvertedIndexConfig != nil {
+ iicm = class.InvertedIndexConfig
+ }
+
+ iics := ConfigFromModel(iicm)
+ sd, err := stopwords.NewDetectorFromConfig(iics.Stopwords)
+ if err != nil {
+ return nil, err
+ }
+
+ return sd, nil
+}
+
type docPointers struct {
count uint64
docIDs []uint64
