write invert batches concurrently
@@ -4,6 +4,7 @@ import (
"bytes"
"encoding/binary"
"fmt"
+ "sync"
"time"
"github.com/boltdb/bolt"
@@ -83,9 +84,13 @@ func (s *Shard) spikeBatchedInverted(objects []*storobj.Object,
}
func (s *Shard) batchExtendInvertedIndices(props map[string]batchProp) error {
+ wg := &sync.WaitGroup{}
for _, prop := range props {
+ wg.Add(1)
+
+ go func(prop batchProp) {
+ defer wg.Done()
- // TODO: try concurrent
before := time.Now()
if err := s.db.Batch(func(tx *bolt.Tx) error {
b := tx.Bucket(helpers.BucketFromPropName(prop.Name))
@@ -114,11 +119,14 @@ func (s *Shard) batchExtendInvertedIndices(props map[string]batchProp) error {
return nil
}); err != nil {
- return errors.Wrap(err, "batch invert props")
+ fmt.Printf("\n\nERROR: %v\n\n", errors.Wrap(err, "batch invert props"))
}
fmt.Printf("prop %s took %s\n", prop.Name, time.Since(before))
+ }(prop)
}
+ wg.Wait()
+
return nil
}
