disksort: save IO by putting in-memory shard directly into merger heap
Reviewers: #core_team, zarko
Subscribers: #core_team
@@ -208,12 +208,9 @@ func (m *mergeSorter) Iterator() (iter Iterator, err error) {
}
}()
- if len(m.buffer) != 0 {
- // To make the merging algorithm simpler, dump the last shard to disk.
- if err := m.dumpShard(); err != nil {
- m.buffer = nil
- return nil, fmt.Errorf("error dumping final shard: %v", err)
- }
+ // Push all of the in-memory elements into the merger heap.
+ for _, el := range m.buffer {
+ heap.Push(merger, &mergeElement{el: el})
}
m.buffer = nil
@@ -270,6 +267,7 @@ func (i *mergeIterator) Next() (interface{}, error) {
x := heap.Pop(i.merger).(*mergeElement)
el := x.el
+ if x.rd != nil {
// Read and parse the next value on the same shard
rec, err := x.rd.Next()
if err != nil {
@@ -288,6 +286,7 @@ func (i *mergeIterator) Next() (interface{}, error) {
x.el = next
heap.Push(i.merger, x)
}
+ }
return el, nil
}
