加入 Gitee
与超过 1200万 开发者一起发现、参与优秀开源项目,私有仓库也完全免费 :)
免费加入
文件
克隆/下载
fastaggregation.go 8.65 KB
一键复制 编辑 原始数据 按行查看 历史
Daniel Lemire 提交于 2020-06-25 17:48 . Fix variable name.
package roaring
import (
"container/heap"
)
// Or function that requires repairAfterLazy
func lazyOR(x1, x2 *Bitmap) *Bitmap {
answer := NewBitmap()
pos1 := 0
pos2 := 0
length1 := x1.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for (pos1 < length1) && (pos2 < length2) {
s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 < s2 {
answer.highlowcontainer.appendCopy(x1.highlowcontainer, pos1)
pos1++
if pos1 == length1 {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
} else if s1 > s2 {
answer.highlowcontainer.appendCopy(x2.highlowcontainer, pos2)
pos2++
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
switch t := c1.(type) {
case *arrayContainer:
c1 = t.toBitmapContainer()
case *runContainer16:
if !t.isFull() {
c1 = t.toBitmapContainer()
}
}
answer.highlowcontainer.appendContainer(s1, c1.lazyOR(x2.highlowcontainer.getContainerAtIndex(pos2)), false)
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
}
if pos1 == length1 {
answer.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
} else if pos2 == length2 {
answer.highlowcontainer.appendCopyMany(x1.highlowcontainer, pos1, length1)
}
return answer
}
// In-place Or function that requires repairAfterLazy
func (x1 *Bitmap) lazyOR(x2 *Bitmap) *Bitmap {
pos1 := 0
pos2 := 0
length1 := x1.highlowcontainer.size()
length2 := x2.highlowcontainer.size()
main:
for (pos1 < length1) && (pos2 < length2) {
s1 := x1.highlowcontainer.getKeyAtIndex(pos1)
s2 := x2.highlowcontainer.getKeyAtIndex(pos2)
for {
if s1 < s2 {
pos1++
if pos1 == length1 {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
} else if s1 > s2 {
x1.highlowcontainer.insertNewKeyValueAt(pos1, s2, x2.highlowcontainer.getContainerAtIndex(pos2).clone())
pos2++
pos1++
length1++
if pos2 == length2 {
break main
}
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
} else {
c1 := x1.highlowcontainer.getContainerAtIndex(pos1)
switch t := c1.(type) {
case *arrayContainer:
c1 = t.toBitmapContainer()
case *runContainer16:
if !t.isFull() {
c1 = t.toBitmapContainer()
}
case *bitmapContainer:
c1 = x1.highlowcontainer.getWritableContainerAtIndex(pos1)
}
x1.highlowcontainer.containers[pos1] = c1.lazyIOR(x2.highlowcontainer.getContainerAtIndex(pos2))
x1.highlowcontainer.needCopyOnWrite[pos1] = false
pos1++
pos2++
if (pos1 == length1) || (pos2 == length2) {
break main
}
s1 = x1.highlowcontainer.getKeyAtIndex(pos1)
s2 = x2.highlowcontainer.getKeyAtIndex(pos2)
}
}
}
if pos1 == length1 {
x1.highlowcontainer.appendCopyMany(x2.highlowcontainer, pos2, length2)
}
return x1
}
// to be called after lazy aggregates
func (x1 *Bitmap) repairAfterLazy() {
for pos := 0; pos < x1.highlowcontainer.size(); pos++ {
c := x1.highlowcontainer.getContainerAtIndex(pos)
switch c.(type) {
case *bitmapContainer:
if c.(*bitmapContainer).cardinality == invalidCardinality {
c = x1.highlowcontainer.getWritableContainerAtIndex(pos)
c.(*bitmapContainer).computeCardinality()
if c.(*bitmapContainer).getCardinality() <= arrayDefaultMaxSize {
x1.highlowcontainer.setContainerAtIndex(pos, c.(*bitmapContainer).toArrayContainer())
} else if c.(*bitmapContainer).isFull() {
x1.highlowcontainer.setContainerAtIndex(pos, newRunContainer16Range(0, MaxUint16))
}
}
}
}
}
// FastAnd computes the intersection between many bitmaps quickly
// Compared to the And function, it can take many bitmaps as input, thus saving the trouble
// of manually calling "And" many times.
func FastAnd(bitmaps ...*Bitmap) *Bitmap {
if len(bitmaps) == 0 {
return NewBitmap()
} else if len(bitmaps) == 1 {
return bitmaps[0].Clone()
}
answer := And(bitmaps[0], bitmaps[1])
for _, bm := range bitmaps[2:] {
answer.And(bm)
}
return answer
}
// FastOr computes the union between many bitmaps quickly, as opposed to having to call Or repeatedly.
// It might also be faster than calling Or repeatedly.
func FastOr(bitmaps ...*Bitmap) *Bitmap {
if len(bitmaps) == 0 {
return NewBitmap()
} else if len(bitmaps) == 1 {
return bitmaps[0].Clone()
}
answer := lazyOR(bitmaps[0], bitmaps[1])
for _, bm := range bitmaps[2:] {
answer = answer.lazyOR(bm)
}
// here is where repairAfterLazy is called.
answer.repairAfterLazy()
return answer
}
// HeapOr computes the union between many bitmaps quickly using a heap.
// It might be faster than calling Or repeatedly.
func HeapOr(bitmaps ...*Bitmap) *Bitmap {
if len(bitmaps) == 0 {
return NewBitmap()
}
// TODO: for better speed, we could do the operation lazily, see Java implementation
pq := make(priorityQueue, len(bitmaps))
for i, bm := range bitmaps {
pq[i] = &item{bm, i}
}
heap.Init(&pq)
for pq.Len() > 1 {
x1 := heap.Pop(&pq).(*item)
x2 := heap.Pop(&pq).(*item)
heap.Push(&pq, &item{Or(x1.value, x2.value), 0})
}
return heap.Pop(&pq).(*item).value
}
// HeapXor computes the symmetric difference between many bitmaps quickly (as opposed to calling Xor repeated).
// Internally, this function uses a heap.
// It might be faster than calling Xor repeatedly.
func HeapXor(bitmaps ...*Bitmap) *Bitmap {
if len(bitmaps) == 0 {
return NewBitmap()
}
pq := make(priorityQueue, len(bitmaps))
for i, bm := range bitmaps {
pq[i] = &item{bm, i}
}
heap.Init(&pq)
for pq.Len() > 1 {
x1 := heap.Pop(&pq).(*item)
x2 := heap.Pop(&pq).(*item)
heap.Push(&pq, &item{Xor(x1.value, x2.value), 0})
}
return heap.Pop(&pq).(*item).value
}
// AndAny provides a result equivalent to x1.And(FastOr(bitmaps)).
// It's optimized to minimize allocations. It also might be faster than separate calls.
func (x1 *Bitmap) AndAny(bitmaps ...*Bitmap) {
if len(bitmaps) == 0 {
return
} else if len(bitmaps) == 1 {
x1.And(bitmaps[0])
return
}
type withPos struct {
bitmap *roaringArray
pos int
key uint16
}
filters := make([]withPos, 0, len(bitmaps))
for _, b := range bitmaps {
if b.highlowcontainer.size() > 0 {
filters = append(filters, withPos{
bitmap: &b.highlowcontainer,
pos: 0,
key: b.highlowcontainer.getKeyAtIndex(0),
})
}
}
basePos := 0
intersections := 0
keyContainers := make([]container, 0, len(filters))
var (
tmpArray *arrayContainer
tmpBitmap *bitmapContainer
minNextKey uint16
)
for basePos < x1.highlowcontainer.size() && len(filters) > 0 {
baseKey := x1.highlowcontainer.getKeyAtIndex(basePos)
// accumulate containers for current key, find next minimal key in filters
// and exclude filters that do not have related values anymore
i := 0
maxPossibleOr := 0
minNextKey = MaxUint16
for _, f := range filters {
if f.key < baseKey {
f.pos = f.bitmap.advanceUntil(baseKey, f.pos)
if f.pos == f.bitmap.size() {
continue
}
f.key = f.bitmap.getKeyAtIndex(f.pos)
}
if f.key == baseKey {
cont := f.bitmap.getContainerAtIndex(f.pos)
keyContainers = append(keyContainers, cont)
maxPossibleOr += cont.getCardinality()
f.pos++
if f.pos == f.bitmap.size() {
continue
}
f.key = f.bitmap.getKeyAtIndex(f.pos)
}
minNextKey = minOfUint16(minNextKey, f.key)
filters[i] = f
i++
}
filters = filters[:i]
if len(keyContainers) == 0 {
basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
continue
}
var ored container
if len(keyContainers) == 1 {
ored = keyContainers[0]
} else {
//TODO: special case for run containers?
if maxPossibleOr > arrayDefaultMaxSize {
if tmpBitmap == nil {
tmpBitmap = newBitmapContainer()
}
tmpBitmap.resetTo(keyContainers[0])
for _, c := range keyContainers[1:] {
tmpBitmap.ior(c)
}
ored = tmpBitmap
} else {
if tmpArray == nil {
tmpArray = newArrayContainerCapacity(maxPossibleOr)
}
tmpArray.realloc(maxPossibleOr)
tmpArray.resetTo(keyContainers[0])
for _, c := range keyContainers[1:] {
tmpArray.ior(c)
}
ored = tmpArray
}
}
result := x1.highlowcontainer.getWritableContainerAtIndex(basePos).iand(ored)
if result.getCardinality() > 0 {
x1.highlowcontainer.replaceKeyAndContainerAtIndex(intersections, baseKey, result, false)
intersections++
}
keyContainers = keyContainers[:0]
basePos = x1.highlowcontainer.advanceUntil(minNextKey, basePos)
}
x1.highlowcontainer.resize(intersections)
}
马建仓 AI 助手
尝试更多
代码解读
代码找茬
代码优化