From ffc39f54627d084cfb0e8a9c4df96ca2e50c2310 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E6=98=94=E5=BF=B5?= <1@72wo.cn> Date: Wed, 2 Jul 2025 22:31:54 +0800 Subject: [PATCH] =?UTF-8?q?refactor(common):=20=E9=87=8D=E6=9E=84=20bitset?= =?UTF-8?q?=20=E5=92=8C=20log=20=E5=8C=85?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - 移除了 github.com/scylladb/termtables 依赖 - 修改了 bitset 包，移除了未使用的代码和测试 - 修改了 log 包，移除了未使用的代码和测试 - 更新了 go.work 文件，添加了 bitset 和 log 包 --- common/go.mod | 1 - .../bitset/.github/workflows/test.yml | 21 + common/serialize/bitset/LICENSE | 28 + common/serialize/bitset/README.md | 68 + common/serialize/bitset/README_zh_CN.md | 66 + common/serialize/bitset/bitset.go | 12 +- common/serialize/bitset/bitset32.go | 963 ++++++++ common/serialize/bitset/bitset32_.go | 43 + .../bitset/bitset32_benchmark_test.go | 454 ++++ common/serialize/bitset/bitset32_test.go | 2011 +++++++++++++++++ common/serialize/bitset/bitset64.go | 46 + common/serialize/bitset/bitset_random_test.go | 174 ++ common/serialize/bitset/go.mod | 5 + common/serialize/bitset/go.sum | 2 + common/serialize/bitset/popcnt_19.go | 43 + common/serialize/bitset/trailingZeroes32.go | 10 + common/serialize/log/.gitignore | 19 + common/serialize/log/.travis.yml | 9 + common/serialize/log/LICENSE | 202 ++ common/serialize/log/README.md | 103 + common/serialize/log/cell.go | 168 ++ common/serialize/log/cell_test.go | 113 + common/serialize/log/go.mod | 8 + common/serialize/log/go.sum | 4 + common/serialize/log/html.go | 107 + common/serialize/log/html_test.go | 222 ++ common/serialize/log/log.go | 5 - common/serialize/log/log_test.go | 19 - common/serialize/log/row.go | 47 + common/serialize/log/row_test.go | 29 + common/serialize/log/separator.go | 60 + common/serialize/log/straight_separator.go | 36 + common/serialize/log/style.go | 214 ++ common/serialize/log/table.go | 373 +++ common/serialize/log/table_test.go | 562 +++++ common/serialize/log/term/env.go | 43 + common/serialize/log/term/getsize.go | 54 + common/serialize/log/term/sizes_unix.go | 35 + common/serialize/log/term/sizes_windows.go | 57 + common/serialize/log/term/wrapper.go | 23 + .../{bitset/bitset_test.go => test_test.go} | 16 +- go.work | 2 + 42 files changed, 6445 insertions(+), 32 deletions(-) create mode 100644 common/serialize/bitset/.github/workflows/test.yml create mode 100644 common/serialize/bitset/LICENSE create mode 100644 common/serialize/bitset/README.md create mode 100644 common/serialize/bitset/README_zh_CN.md create mode 100644 common/serialize/bitset/bitset32.go create mode 100644 common/serialize/bitset/bitset32_.go create mode 100644 common/serialize/bitset/bitset32_benchmark_test.go create mode 100644 common/serialize/bitset/bitset32_test.go create mode 100644 common/serialize/bitset/bitset64.go create mode 100644 common/serialize/bitset/bitset_random_test.go create mode 100644 common/serialize/bitset/go.mod create mode 100644 common/serialize/bitset/go.sum create mode 100644 common/serialize/bitset/popcnt_19.go create mode 100644 common/serialize/bitset/trailingZeroes32.go create mode 100644 common/serialize/log/.gitignore create mode 100644 common/serialize/log/.travis.yml create mode 100644 common/serialize/log/LICENSE create mode 100644 common/serialize/log/README.md create mode 100644 common/serialize/log/cell.go create mode 100644 common/serialize/log/cell_test.go create mode 100644 common/serialize/log/go.mod create mode 100644 common/serialize/log/go.sum create mode 100644 common/serialize/log/html.go create mode 100644 common/serialize/log/html_test.go delete mode 100644 common/serialize/log/log.go delete mode 100644 common/serialize/log/log_test.go create mode 100644 common/serialize/log/row.go create mode 100644 common/serialize/log/row_test.go create mode 100644 common/serialize/log/separator.go create mode 100644 common/serialize/log/straight_separator.go create mode 100644 common/serialize/log/style.go create mode 100644 common/serialize/log/table.go create mode 100644 common/serialize/log/table_test.go create mode 100644 common/serialize/log/term/env.go create mode 100644 common/serialize/log/term/getsize.go create mode 100644 common/serialize/log/term/sizes_unix.go create mode 100644 common/serialize/log/term/sizes_windows.go create mode 100644 common/serialize/log/term/wrapper.go rename common/serialize/{bitset/bitset_test.go => test_test.go} (69%) diff --git a/common/go.mod b/common/go.mod index 0193b971d..52033fd49 100644 --- a/common/go.mod +++ b/common/go.mod @@ -19,7 +19,6 @@ require ( github.com/go-redis/redis/v8 v8.11.5 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/pointernil/bitset32 v0.0.1 // indirect - github.com/scylladb/termtables v1.0.0 // indirect github.com/yitter/idgenerator-go v1.3.3 // indirect ) diff --git a/common/serialize/bitset/.github/workflows/test.yml b/common/serialize/bitset/.github/workflows/test.yml new file mode 100644 index 000000000..a03495aad --- /dev/null +++ b/common/serialize/bitset/.github/workflows/test.yml @@ -0,0 +1,21 @@ +name: Test + +on: [push, pull_request] +permissions: + contents: read +jobs: + test: + strategy: + matrix: + go-version: [1.19.x] + os: [ubuntu-latest, macos-latest, windows-latest] + runs-on: ${{ matrix.os }} + steps: + - name: Install Go + uses: actions/setup-go@v2 + with: + go-version: ${{ matrix.go-version }} + - name: Checkout code + uses: actions/checkout@master + - name: Test + run: go test ./... \ No newline at end of file diff --git a/common/serialize/bitset/LICENSE b/common/serialize/bitset/LICENSE new file mode 100644 index 000000000..6dc2952ee --- /dev/null +++ b/common/serialize/bitset/LICENSE @@ -0,0 +1,28 @@ +BSD 3-Clause License + +Copyright (c) 2023, Nil@Pointer + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/common/serialize/bitset/README.md b/common/serialize/bitset/README.md new file mode 100644 index 000000000..a0cc8f31e --- /dev/null +++ b/common/serialize/bitset/README.md @@ -0,0 +1,68 @@ +# bitset32 + +[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/pointernil/bitset32/actions?query=workflow%3ATest) + +[zh_CN 简体中文](./README_zh_CN.md) + +## Description + +Package bitset32 modified from `"github.com/bits-and-blooms/bitset"` +implements bitset with uint32. Both packages are used in the same way. + +If not necessary, it is highly recommended to use +["github.com/bits-and-blooms/bitset"](https://github.com/bits-and-blooms/bitset). + +## Go version +``` +go version go1.19.4 windows/amd64 +``` + +## Install +``` +go get github.com/pointernil/bitset32 +``` + +## Testing +``` +go test +go test -cover +``` + +## Usage +``` +package main + +import ( + "fmt" + "math/rand" + + "github.com/pointernil/bitset32" +) + +func main() { + fmt.Printf("Hello from BitSet!\n") + var b bitset32.BitSet32 + // play some Go Fish + for i := 0; i < 100; i++ { + card1 := uint(rand.Intn(52)) + card2 := uint(rand.Intn(52)) + b.Set(card1) + if b.Test(card2) { + fmt.Println("Go Fish!") + } + b.Clear(card1) + } + + // Chaining + b.Set(10).Set(11) + + for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { + fmt.Println("The following bit is set:", i) + } + if b.Intersection(bitset32.New(100).Set(10)).Count() == 1 { + fmt.Println("Intersection works.") + } else { + fmt.Println("Intersection doesn't work???") + } +} +``` \ No newline at end of file diff --git a/common/serialize/bitset/README_zh_CN.md b/common/serialize/bitset/README_zh_CN.md new file mode 100644 index 000000000..37e9ad64e --- /dev/null +++ b/common/serialize/bitset/README_zh_CN.md @@ -0,0 +1,66 @@ +# bitset32 + +[![Test](https://github.com/bits-and-blooms/bitset/workflows/Test/badge.svg)](https://github.com/pointernil/bitset32/actions?query=workflow%3ATest) + +[en English](./README.md) + +## 简介 + +包 `bitset32` 修改自 `"github.com/bits-and-blooms/bitset"`，底层使用uint32存数据。`bitset32` 与 `bitset` 用法一致。 + +如非必要，请使用 ["github.com/bits-and-blooms/bitset"](https://github.com/bits-and-blooms/bitset)。 + +## Golang版本 +``` +go version go1.19.4 windows/amd64 +``` + +## 安装 +``` +go get github.com/pointernil/bitset32 +``` + +## 测试 +``` +go test +go test -cover +``` + +## 使用示意 +``` +package main + +import ( + "fmt" + "math/rand" + + "github.com/pointernil/bitset32" +) + +func main() { + fmt.Printf("! \n") + var b bitset32.BitSet32 + // play some Go Fish + for i := 0; i < 100; i++ { + card1 := uint(rand.Intn(52)) + card2 := uint(rand.Intn(52)) + b.Set(card1) + if b.Test(card2) { + fmt.Println("Go Fish!") + } + b.Clear(card1) + } + + // Chaining + b.Set(10).Set(11) + + for i, e := b.NextSet(0); e; i, e = b.NextSet(i + 1) { + fmt.Println("The following bit is set:", i) + } + if b.Intersection(bitset32.New(100).Set(10)).Count() == 1 { + fmt.Println("Intersection works.") + } else { + fmt.Println("Intersection doesn't work???") + } +} +``` \ No newline at end of file diff --git a/common/serialize/bitset/bitset.go b/common/serialize/bitset/bitset.go index 302da08fe..b978466de 100644 --- a/common/serialize/bitset/bitset.go +++ b/common/serialize/bitset/bitset.go @@ -1,5 +1,7 @@ -package bitset - -func teset() { - -} +// Package bitset32 modified from "github.com/bits-and-blooms/bitset" +// implements bitset with uint32. Both packages are used in the same way. +// In bitset32, some methods are untested. So if not necessary, +// it is highly recommended to use "github.com/bits-and-blooms/bitset". + +// go version go1.19.4 windows/amd64 +package bitset32 diff --git a/common/serialize/bitset/bitset32.go b/common/serialize/bitset/bitset32.go new file mode 100644 index 000000000..3d9c033c0 --- /dev/null +++ b/common/serialize/bitset/bitset32.go @@ -0,0 +1,963 @@ +/* +Package bitset implements bitsets, a mapping +between non-negative integers and boolean values. It should be more +efficient than map[uint] bool. + +It provides methods for setting, clearing, flipping, and testing +individual integers. + +But it also provides set intersection, union, difference, +complement, and symmetric operations, as well as tests to +check whether any, all, or no bits are set, and querying a +bitset's current length and number of positive bits. + +BitSets are expanded to the size of the largest set bit; the +memory allocation is approximately Max bits, where Max is +the largest set bit. BitSets are never shrunk. On creation, +a hint can be given for the number of bits that will be used. + +Many of the methods, including Set,Clear, and Flip, return +a BitSet pointer, which allows for chaining. + +Example use: + + import "bitset" + var b BitSet + b.Set(10).Set(11) + if b.Test(1000) { + b.Clear(1000) + } + if B.Intersection(bitset.New(100).Set(10)).Count() > 1 { + fmt.Println("Intersection works.") + } + +As an alternative to BitSets, one should check out the 'big' package, +which provides a (less set-theoretical) view of bitsets. +*/ +package bitset32 + +import ( + "bufio" + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + "math/bits" + "strconv" +) + +// the wordSize of a bit set +const wordSize = uint(32) + +// log2WordSize is lg(wordSize) +const log2WordSize = uint(5) + +// allBits has every bit set +const allBits uint32 = 0xffffffff + +// TODO BUGFIX +// default binary BigEndian +var binaryOrder binary.ByteOrder = binary.BigEndian + +// A BitSet is a set of bits. The zero value of a BitSet is an empty set of length 0. +type BitSet32 struct { + length uint + set []uint32 +} + +// Error is used to distinguish errors (panics) generated in this package. +type Error string + +// safeSet will fixup b.set to be non-nil and return the field value +func (b *BitSet32) safeSet() []uint32 { + if b.set == nil { + b.set = make([]uint32, wordsNeeded(0)) + } + return b.set +} + +// SetBitsetFrom fills the bitset with an array of integers without creating a new BitSet instance +func (b *BitSet32) SetBitsetFrom(buf []uint32) { + b.length = uint(len(buf)) * 32 + b.set = buf +} + +// From is a constructor used to create a BitSet from an array of integers +func From(buf []uint32) *BitSet32 { + return FromWithLength(uint(len(buf))*32, buf) +} + +// FromWithLength constructs from an array of integers and length. +func FromWithLength(len uint, set []uint32) *BitSet32 { + return &BitSet32{len, set} +} + +// Bytes returns the bitset as array of integers +func (b *BitSet32) Bytes() []uint32 { + return b.set +} + +// wordsNeeded calculates the number of words needed for i bits +func wordsNeeded(i uint) int { + if i > (Cap() - wordSize + 1) { + return int(Cap() >> log2WordSize) + } + return int((i + (wordSize - 1)) >> log2WordSize) +} + +// wordsNeededUnbound calculates the number of words needed for i bits, possibly exceeding the capacity. +// This function is useful if you know that the capacity cannot be exceeded (e.g., you have an existing bitmap). +func wordsNeededUnbound(i uint) int { + return int((i + (wordSize - 1)) >> log2WordSize) +} + +// wordsIndex calculates the index of words in a `uint64` +func wordsIndex(i uint) uint { + return i & (wordSize - 1) +} + +// New creates a new BitSet with a hint that length bits will be required +func New(length uint) (bset *BitSet32) { + defer func() { + if r := recover(); r != nil { + bset = &BitSet32{ + 0, + make([]uint32, 0), + } + } + }() + + bset = &BitSet32{ + length, + make([]uint32, wordsNeeded(length)), + } + + return bset +} + +// Cap returns the total possible capacity, or number of bits +func Cap() uint { + return ^uint(0) +} + +// Len returns the number of bits in the BitSet. +// Note the difference to method Count, see example. +func (b *BitSet32) Len() uint { + return b.length +} + +// extendSet adds additional words to incorporate new bits if needed +func (b *BitSet32) extendSet(i uint) { + if i >= Cap() { + panic("You are exceeding the capacity") + } + nsize := wordsNeeded(i + 1) + if b.set == nil { + b.set = make([]uint32, nsize) + } else if cap(b.set) >= nsize { + b.set = b.set[:nsize] // fast resize + } else if len(b.set) < nsize { + newset := make([]uint32, nsize, 2*nsize) // increase capacity 2x + copy(newset, b.set) + b.set = newset + } + b.length = i + 1 +} + +// Test whether bit i is set. +func (b *BitSet32) Test(i uint) bool { + if i >= b.length { + return false + } + return b.set[i>>log2WordSize]&(1<= Cap(), this function will panic. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet32) Set(i uint) *BitSet32 { + if i >= b.length { // if we need more bits, make 'em + b.extendSet(i) + } + b.set[i>>log2WordSize] |= 1 << wordsIndex(i) + return b +} + +// Clear bit i to 0 +func (b *BitSet32) Clear(i uint) *BitSet32 { + if i >= b.length { + return b + } + b.set[i>>log2WordSize] &^= 1 << wordsIndex(i) + return b +} + +// SetTo sets bit i to value. +// If i>= Cap(), this function will panic. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet32) SetTo(i uint, value bool) *BitSet32 { + if value { + return b.Set(i) + } + return b.Clear(i) +} + +// Flip bit at i. +// If i>= Cap(), this function will panic. +// Warning: using a very large value for 'i' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet32) Flip(i uint) *BitSet32 { + if i >= b.length { + return b.Set(i) + } + b.set[i>>log2WordSize] ^= 1 << wordsIndex(i) + return b +} + +// FlipRange bit in [start, end). +// If end>= Cap(), this function will panic. +// Warning: using a very large value for 'end' +// may lead to a memory shortage and a panic: the caller is responsible +// for providing sensible parameters in line with their memory capacity. +func (b *BitSet32) FlipRange(start, end uint) *BitSet32 { + if start >= end { + return b + } + if end-1 >= b.length { // if we need more bits, make 'em + b.extendSet(end - 1) + } + var startWord uint = start >> log2WordSize + var endWord uint = end >> log2WordSize + b.set[startWord] ^= ^(^uint32(0) << wordsIndex(start)) + for i := startWord; i < endWord; i++ { + b.set[i] = ^b.set[i] + } + if end&(wordSize-1) != 0 { + b.set[endWord] ^= ^uint32(0) >> wordsIndex(-end) + } + return b +} + +// Shrink shrinks BitSet so that the provided value is the last possible +// set value. It clears all bits > the provided index and reduces the size +// and length of the set. +// +// Note that the parameter value is not the new length in bits: it is the +// maximal value that can be stored in the bitset after the function call. +// The new length in bits is the parameter value + 1. Thus it is not possible +// to use this function to set the length to 0, the minimal value of the length +// after this function call is 1. +// +// A new slice is allocated to store the new bits, so you may see an increase in +// memory usage until the GC runs. Normally this should not be a problem, but if you +// have an extremely large BitSet its important to understand that the old BitSet will +// remain in memory until the GC frees it. +func (b *BitSet32) Shrink(lastbitindex uint) *BitSet32 { + length := lastbitindex + 1 + idx := wordsNeeded(length) + if idx > len(b.set) { + return b + } + shrunk := make([]uint32, idx) + copy(shrunk, b.set[:idx]) + b.set = shrunk + b.length = length + lastWordUsedBits := length % 32 + if lastWordUsedBits != 0 { + b.set[idx-1] &= allBits >> uint32(32-wordsIndex(lastWordUsedBits)) + } + return b +} + +// Compact shrinks BitSet to so that we preserve all set bits, while minimizing +// memory usage. Compact calls Shrink. +func (b *BitSet32) Compact() *BitSet32 { + idx := len(b.set) - 1 + for ; idx >= 0 && b.set[idx] == 0; idx-- { + } + newlength := uint((idx + 1) << log2WordSize) + if newlength >= b.length { + return b // nothing to do + } + if newlength > 0 { + return b.Shrink(newlength - 1) + } + // TODO: FIX + // We preserve one word + return b.Shrink(31) +} + +// InsertAt takes an index which indicates where a bit should be +// inserted. Then it shifts all the bits in the set to the left by 1, starting +// from the given index position, and sets the index position to 0. +// +// Depending on the size of your BitSet, and where you are inserting the new entry, +// this method could be extremely slow and in some cases might cause the entire BitSet +// to be recopied. +func (b *BitSet32) InsertAt(idx uint) *BitSet32 { + insertAtElement := idx >> log2WordSize + + // if length of set is a multiple of wordSize we need to allocate more space first + if b.isLenExactMultiple() { + b.set = append(b.set, uint32(0)) + } + + var i uint + for i = uint(len(b.set) - 1); i > insertAtElement; i-- { + // all elements above the position where we want to insert can simply by shifted + b.set[i] <<= 1 + + // we take the most significant bit of the previous element and set it as + // the least significant bit of the current element + // TODO: FIX + b.set[i] |= (b.set[i-1] & 0x80000000) >> 31 + } + + // generate a mask to extract the data that we need to shift left + // within the element where we insert a bit + dataMask := uint32(1)< 0x40000 { + buffer.WriteString("...") + break + } + buffer.WriteString(strconv.FormatInt(int64(i), 10)) + i, e = b.NextSet(i + 1) + if e { + buffer.WriteString(",") + } + } + buffer.WriteString("}") + return buffer.String() +} + +// DeleteAt deletes the bit at the given index position from +// within the bitset +// All the bits residing on the left of the deleted bit get +// shifted right by 1 +// The running time of this operation may potentially be +// relatively slow, O(length) +func (b *BitSet32) DeleteAt(i uint) *BitSet32 { + // the index of the slice element where we'll delete a bit + deleteAtElement := i >> log2WordSize + + // generate a mask for the data that needs to be shifted right + // within that slice element that gets modified + dataMask := ^((uint32(1) << wordsIndex(i)) - 1) + + // extract the data that we'll shift right from the slice element + data := b.set[deleteAtElement] & dataMask + + // set the masked area to 0 while leaving the rest as it is + b.set[deleteAtElement] &= ^dataMask + + // shift the previously extracted data to the right and then + // set it in the previously masked area + b.set[deleteAtElement] |= (data >> 1) & dataMask + + // loop over all the consecutive slice elements to copy each + // lowest bit into the highest position of the previous element, + // then shift the entire content to the right by 1 + for i := int(deleteAtElement) + 1; i < len(b.set); i++ { + b.set[i-1] |= (b.set[i] & 1) << 31 + b.set[i] >>= 1 + } + + b.length = b.length - 1 + + return b +} + +// NextSet returns the next bit set from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no set bit found) +// for i,e := v.NextSet(0); e; i,e = v.NextSet(i + 1) {...} +// +// Users concerned with performance may want to use NextSetMany to +// retrieve several values at once. +func (b *BitSet32) NextSet(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> wordsIndex(i) + if w != 0 { + return i + uint(bits.TrailingZeros32(w)), true + } + x = x + 1 + for x < len(b.set) { + if b.set[x] != 0 { + return uint(x)*wordSize + uint(bits.TrailingZeros32(b.set[x])), true + } + x = x + 1 + + } + return 0, false +} + +// NextSetMany returns many next bit sets from the specified index, +// including possibly the current index and up to cap(buffer). +// If the returned slice has len zero, then no more set bits were found +// +// buffer := make([]uint, 256) // this should be reused +// j := uint(0) +// j, buffer = bitmap.NextSetMany(j, buffer) +// for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j,buffer) { +// for k := range buffer { +// do something with buffer[k] +// } +// j += 1 +// } +// +// It is possible to retrieve all set bits as follow: +// +// indices := make([]uint, bitmap.Count()) +// bitmap.NextSetMany(0, indices) +// +// However if bitmap.Count() is large, it might be preferable to +// use several calls to NextSetMany, for performance reasons. +func (b *BitSet32) NextSetMany(i uint, buffer []uint) (uint, []uint) { + myanswer := buffer + capacity := cap(buffer) + x := int(i >> log2WordSize) + if x >= len(b.set) || capacity == 0 { + return 0, myanswer[:0] + } + skip := wordsIndex(i) + word := b.set[x] >> skip + myanswer = myanswer[:capacity] + size := int(0) + for word != 0 { + r := uint(bits.TrailingZeros32(word)) + t := word & ((^word) + 1) + myanswer[size] = r + i + size++ + if size == capacity { + goto End + } + word = word ^ t + } + x++ + for idx, word := range b.set[x:] { + for word != 0 { + r := uint(bits.TrailingZeros32(word)) + t := word & ((^word) + 1) + myanswer[size] = r + (uint(x+idx) << 6) + size++ + if size == capacity { + goto End + } + word = word ^ t + } + } +End: + if size > 0 { + return myanswer[size-1], myanswer[:size] + } + return 0, myanswer[:0] +} + +// NextClear returns the next clear bit from the specified index, +// including possibly the current index +// along with an error code (true = valid, false = no bit found i.e. all bits are set) +func (b *BitSet32) NextClear(i uint) (uint, bool) { + x := int(i >> log2WordSize) + if x >= len(b.set) { + return 0, false + } + w := b.set[x] + w = w >> wordsIndex(i) + wA := allBits >> wordsIndex(i) + index := i + uint(bits.TrailingZeros32(^w)) + if w != wA && index < b.length { + return index, true + } + x++ + for x < len(b.set) { + index = uint(x)*wordSize + uint(bits.TrailingZeros32(^b.set[x])) + if b.set[x] != allBits && index < b.length { + return index, true + } + x++ + } + return 0, false +} + +// ClearAll clears the entire BitSet +func (b *BitSet32) ClearAll() *BitSet32 { + if b != nil && b.set != nil { + for i := range b.set { + b.set[i] = 0 + } + } + return b +} + +// wordCount returns the number of words used in a bit set +func (b *BitSet32) wordCount() int { + return wordsNeededUnbound(b.length) +} + +// Clone this BitSet +func (b *BitSet32) Clone() *BitSet32 { + c := New(b.length) + if b.set != nil { // Clone should not modify current object + copy(c.set, b.set) + } + return c +} + +// Copy into a destination BitSet using the Go array copy semantics: +// the number of bits copied is the minimum of the number of bits in the current +// BitSet (Len()) and the destination Bitset. +// We return the number of bits copied in the destination BitSet. +func (b *BitSet32) Copy(c *BitSet32) (count uint) { + if c == nil { + return + } + if b.set != nil { // Copy should not modify current object + copy(c.set, b.set) + } + count = c.length + if b.length < c.length { + count = b.length + } + // Cleaning the last word is needed to keep the invariant that other functions, such as Count, require + // that any bits in the last word that would exceed the length of the bitmask are set to 0. + c.cleanLastWord() + return +} + +// CopyFull copies into a destination BitSet such that the destination is +// identical to the source after the operation, allocating memory if necessary. +func (b *BitSet32) CopyFull(c *BitSet32) { + if c == nil { + return + } + c.length = b.length + if len(b.set) == 0 { + if c.set != nil { + c.set = c.set[:0] + } + } else { + if cap(c.set) < len(b.set) { + c.set = make([]uint32, len(b.set)) + } else { + c.set = c.set[:len(b.set)] + } + copy(c.set, b.set) + } +} + +// Count (number of set bits). +// Also known as "popcount" or "population count". +func (b *BitSet32) Count() uint { + if b != nil && b.set != nil { + return uint(popcntSlice(b.set)) + } + return 0 +} + +// Equal tests the equivalence of two BitSets. +// False if they are of different sizes, otherwise true +// only if all the same bits are set +func (b *BitSet32) Equal(c *BitSet32) bool { + if c == nil || b == nil { + return c == b + } + if b.length != c.length { + return false + } + if b.length == 0 { // if they have both length == 0, then could have nil set + return true + } + wn := b.wordCount() + for p := 0; p < wn; p++ { + if c.set[p] != b.set[p] { + return false + } + } + return true +} + +func panicIfNull(b *BitSet32) { + if b == nil { + panic(Error("BitSet must not be null")) + } +} + +// Difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet32) Difference(compare *BitSet32) (result *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + result = b.Clone() // clone b (in case b is bigger than compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + result.set[i] = b.set[i] &^ compare.set[i] + } + return +} + +// DifferenceCardinality computes the cardinality of the differnce +func (b *BitSet32) DifferenceCardinality(compare *BitSet32) uint { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + cnt := uint64(0) + cnt += popcntMaskSlice(b.set[:l], compare.set[:l]) + cnt += popcntSlice(b.set[l:]) + return uint(cnt) +} + +// InPlaceDifference computes the difference of base set and other set +// This is the BitSet equivalent of &^ (and not) +func (b *BitSet32) InPlaceDifference(compare *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &^= compare.set[i] + } +} + +// Convenience function: return two bitsets ordered by +// increasing length. Note: neither can be nil +func sortByLength(a *BitSet32, b *BitSet32) (ap *BitSet32, bp *BitSet32) { + if a.length <= b.length { + ap, bp = a, b + } else { + ap, bp = b, a + } + return +} + +// Intersection of base set and other set +// This is the BitSet equivalent of & (and) +func (b *BitSet32) Intersection(compare *BitSet32) (result *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = New(b.length) + for i, word := range b.set { + result.set[i] = word & compare.set[i] + } + return +} + +// IntersectionCardinality computes the cardinality of the union +func (b *BitSet32) IntersectionCardinality(compare *BitSet32) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntAndSlice(b.set, compare.set) + return uint(cnt) +} + +// InPlaceIntersection destructively computes the intersection of +// base set and the compare set. +// This is the BitSet equivalent of & (and) +func (b *BitSet32) InPlaceIntersection(compare *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + for i := 0; i < l; i++ { + b.set[i] &= compare.set[i] + } + for i := l; i < len(b.set); i++ { + b.set[i] = 0 + } + if compare.length > 0 { + if compare.length-1 >= b.length { + b.extendSet(compare.length - 1) + } + } +} + +// Union of base set and other set +// This is the BitSet equivalent of | (or) +func (b *BitSet32) Union(compare *BitSet32) (result *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word | compare.set[i] + } + return +} + +// UnionCardinality computes the cardinality of the uniton of the base set +// and the compare set. +func (b *BitSet32) UnionCardinality(compare *BitSet32) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntOrSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceUnion creates the destructive union of base set and compare set. +// This is the BitSet equivalent of | (or). +func (b *BitSet32) InPlaceUnion(compare *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 && compare.length-1 >= b.length { + b.extendSet(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] |= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet32) SymmetricDifference(compare *BitSet32) (result *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + // compare is bigger, so clone it + result = compare.Clone() + for i, word := range b.set { + result.set[i] = word ^ compare.set[i] + } + return +} + +// SymmetricDifferenceCardinality computes the cardinality of the symmetric difference +func (b *BitSet32) SymmetricDifferenceCardinality(compare *BitSet32) uint { + panicIfNull(b) + panicIfNull(compare) + b, compare = sortByLength(b, compare) + cnt := popcntXorSlice(b.set, compare.set) + if len(compare.set) > len(b.set) { + cnt += popcntSlice(compare.set[len(b.set):]) + } + return uint(cnt) +} + +// InPlaceSymmetricDifference creates the destructive SymmetricDifference of base set and other set +// This is the BitSet equivalent of ^ (xor) +func (b *BitSet32) InPlaceSymmetricDifference(compare *BitSet32) { + panicIfNull(b) + panicIfNull(compare) + l := int(compare.wordCount()) + if l > int(b.wordCount()) { + l = int(b.wordCount()) + } + if compare.length > 0 && compare.length-1 >= b.length { + b.extendSet(compare.length - 1) + } + for i := 0; i < l; i++ { + b.set[i] ^= compare.set[i] + } + if len(compare.set) > l { + for i := l; i < len(compare.set); i++ { + b.set[i] = compare.set[i] + } + } +} + +// Is the length an exact multiple of word sizes? +func (b *BitSet32) isLenExactMultiple() bool { + return wordsIndex(b.length) == 0 +} + +// Clean last word by setting unused bits to 0 +func (b *BitSet32) cleanLastWord() { + if !b.isLenExactMultiple() { + b.set[len(b.set)-1] &= allBits >> (wordSize - wordsIndex(b.length)) + } +} + +// Complement computes the (local) complement of a bitset (up to length bits) +func (b *BitSet32) Complement() (result *BitSet32) { + panicIfNull(b) + result = New(b.length) + for i, word := range b.set { + result.set[i] = ^word + } + result.cleanLastWord() + return +} + +// All returns true if all bits are set, false otherwise. Returns true for +// empty sets. +func (b *BitSet32) All() bool { + panicIfNull(b) + return b.Count() == b.length +} + +// None returns true if no bit is set, false otherwise. Returns true for +// empty sets. +func (b *BitSet32) None() bool { + panicIfNull(b) + if b != nil && b.set != nil { + for _, word := range b.set { + if word > 0 { + return false + } + } + } + return true +} + +// Any returns true if any bit is set, false otherwise +func (b *BitSet32) Any() bool { + panicIfNull(b) + return !b.None() +} + +// IsSuperSet returns true if this is a superset of the other set +func (b *BitSet32) IsSuperSet(other *BitSet32) bool { + for i, e := other.NextSet(0); e; i, e = other.NextSet(i + 1) { + if !b.Test(i) { + return false + } + } + return true +} + +// IsStrictSuperSet returns true if this is a strict superset of the other set +func (b *BitSet32) IsStrictSuperSet(other *BitSet32) bool { + return b.Count() > other.Count() && b.IsSuperSet(other) +} + +// DumpAsBits dumps a bit set as a string of bits +func (b *BitSet32) DumpAsBits() string { + if b.set == nil { + return "." + } + buffer := bytes.NewBufferString("") + i := len(b.set) - 1 + for ; i >= 0; i-- { + fmt.Fprintf(buffer, "%064b.", b.set[i]) + } + return buffer.String() +} + +// BinaryStorageSize returns the binary storage requirements +func (b *BitSet32) BinaryStorageSize() int { + nWords := b.wordCount() + return binary.Size(uint64(0)) + binary.Size(b.set[:nWords]) +} + +// WriteTo writes a BitSet to a stream +func (b *BitSet32) WriteTo(stream io.Writer) (int64, error) { + length := uint64(b.length) + + // Write length + err := binary.Write(stream, binaryOrder, length) + if err != nil { + return 0, err + } + + // Write set + // current implementation of bufio.Writer is more memory efficient than + // binary.Write for large set + writer := bufio.NewWriter(stream) + var item = make([]byte, binary.Size(uint32(0))) // for serializing one uint32 + nWords := b.wordCount() + for i := range b.set[:nWords] { + binaryOrder.PutUint32(item, b.set[i]) + if nn, err := writer.Write(item); err != nil { + return int64(i*binary.Size(uint32(0)) + nn), err + } + } + + err = writer.Flush() + return int64(b.BinaryStorageSize()), err +} + +// ReadFrom reads a BitSet from a stream written using WriteTo +func (b *BitSet32) ReadFrom(stream io.Reader) (int64, error) { + var length uint64 + + // Read length first + err := binary.Read(stream, binaryOrder, &length) + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, err + } + newset := New(uint(length)) + + if uint64(newset.length) != length { + return 0, errors.New("unmarshalling error: type mismatch") + } + + var item [4]byte + nWords := wordsNeeded(uint(length)) + reader := bufio.NewReader(io.LimitReader(stream, 4*int64(nWords))) + for i := 0; i < nWords; i++ { + if _, err := io.ReadFull(reader, item[:]); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + return 0, err + } + newset.set[i] = binaryOrder.Uint32(item[:]) + } + + *b = *newset + return int64(b.BinaryStorageSize()), nil +} diff --git a/common/serialize/bitset/bitset32_.go b/common/serialize/bitset/bitset32_.go new file mode 100644 index 000000000..13616dd4e --- /dev/null +++ b/common/serialize/bitset/bitset32_.go @@ -0,0 +1,43 @@ +package bitset32 + +// MaxConsecutiveOne +func (b *BitSet32) MaxConsecutiveOne(start, end uint) uint { + return b.consecutiveMaxCount(start, end, true) +} + +// MaxConsecutiveZero +func (b *BitSet32) MaxConsecutiveZero(start, end uint) uint { + return b.consecutiveMaxCount(start, end, false) +} + +func (b *BitSet32) consecutiveMaxCount(start, end uint, flag bool) uint { + flag = !flag + if end > b.Len() { + end = b.Len() + } + if start >= b.Len() { + return 0 + } + if start > end { + return 0 + } + rt, sum := uint(0), uint(0) + for i := start; i < end; i++ { + if xor(flag, b.Test(i)) { + sum++ + continue + } + if sum > rt { + rt = sum + } + sum = 0 + } + if sum > rt { + rt = sum + } + return rt +} + +func xor(a, b bool) bool { + return (a || b) && !(a && b) +} diff --git a/common/serialize/bitset/bitset32_benchmark_test.go b/common/serialize/bitset/bitset32_benchmark_test.go new file mode 100644 index 000000000..ca158c3af --- /dev/null +++ b/common/serialize/bitset/bitset32_benchmark_test.go @@ -0,0 +1,454 @@ +// Copyright 2014 Will Fitzgerald. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file tests bit sets + +package bitset32 + +import ( + "math/rand" + "testing" +) + +func BenchmarkSet(b *testing.B) { + b.StopTimer() + r := rand.New(rand.NewSource(0)) + sz := 100000 + s := New(uint(sz)) + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Set(uint(r.Int31n(int32(sz)))) + } +} + +func BenchmarkGetTest(b *testing.B) { + b.StopTimer() + r := rand.New(rand.NewSource(0)) + sz := 100000 + s := New(uint(sz)) + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Test(uint(r.Int31n(int32(sz)))) + } +} + +func BenchmarkSetExpand(b *testing.B) { + b.StopTimer() + sz := uint(100000) + b.StartTimer() + for i := 0; i < b.N; i++ { + var s BitSet32 + s.Set(sz) + } +} + +// go test -bench=Count +func BenchmarkCount(b *testing.B) { + b.StopTimer() + s := New(100000) + for i := 0; i < 100000; i += 100 { + s.Set(uint(i)) + } + b.StartTimer() + for i := 0; i < b.N; i++ { + s.Count() + } +} + +// go test -bench=Iterate +func BenchmarkIterate(b *testing.B) { + b.StopTimer() + s := New(10000) + for i := 0; i < 10000; i += 3 { + s.Set(uint(i)) + } + b.StartTimer() + for j := 0; j < b.N; j++ { + c := uint(0) + for i, e := s.NextSet(0); e; i, e = s.NextSet(i + 1) { + c++ + } + } +} + +// go test -bench=SparseIterate +func BenchmarkSparseIterate(b *testing.B) { + b.StopTimer() + s := New(100000) + for i := 0; i < 100000; i += 30 { + s.Set(uint(i)) + } + b.StartTimer() + for j := 0; j < b.N; j++ { + c := uint(0) + for i, e := s.NextSet(0); e; i, e = s.NextSet(i + 1) { + c++ + } + } +} + +// go test -bench=LemireCreate +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-BitSet32-performance-test/ +func BenchmarkLemireCreate(b *testing.B) { + for i := 0; i < b.N; i++ { + bitmap := New(0) // we force dynamic memory allocation + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + } +} + +// go test -bench=LemireCount +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-BitSet32-performance-test/ +func BenchmarkLemireCount(b *testing.B) { + bitmap := New(100000000) + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + sum += bitmap.Count() + } + if sum == 0 { // added just to fool ineffassign + return + } +} + +// go test -bench=LemireIterate +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-BitSet32-performance-test/ +func BenchmarkLemireIterate(b *testing.B) { + bitmap := New(100000000) + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + for j, e := bitmap.NextSet(0); e; j, e = bitmap.NextSet(j + 1) { + sum++ + } + } + if sum == 0 { // added just to fool ineffassign + return + } +} + +// go test -bench=LemireIterateb +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-BitSet32-performance-test/ +func BenchmarkLemireIterateb(b *testing.B) { + bitmap := New(100000000) + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + for j, e := bitmap.NextSet(0); e; j, e = bitmap.NextSet(j + 1) { + sum += j + } + } + + if sum == 0 { // added just to fool ineffassign + return + } +} + +// go test -bench=BenchmarkLemireIterateManyb +// see http://lemire.me/blog/2016/09/22/swift-versus-java-the-BitSet32-performance-test/ +func BenchmarkLemireIterateManyb(b *testing.B) { + bitmap := New(100000000) + for v := uint(0); v <= 100000000; v += 100 { + bitmap.Set(v) + } + buffer := make([]uint, 256) + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + j := uint(0) + j, buffer = bitmap.NextSetMany(j, buffer) + for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j, buffer) { + for k := range buffer { + sum += buffer[k] + } + j++ + } + } + + if sum == 0 { // added just to fool ineffassign + return + } +} + +func setRnd(bits []uint32, halfings int) { + var rndsource = rand.NewSource(0) + var rnd = rand.New(rndsource) + for i := range bits { + bits[i] = 0xFFFFFFFF + for j := 0; j < halfings; j++ { + bits[i] &= rnd.Uint32() + } + } +} + +// go test -bench=BenchmarkFlorianUekermannIterateMany +func BenchmarkFlorianUekermannIterateMany(b *testing.B) { + var input = make([]uint32, 68) + setRnd(input, 4) + var bitmap = From(input) + buffer := make([]uint, 256) + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + var last, batch = bitmap.NextSetMany(0, buffer) + for len(batch) > 0 { + for _, idx := range batch { + checksum += idx + } + last, batch = bitmap.NextSetMany(last+1, batch) + } + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannIterateManyReg(b *testing.B) { + var input = make([]uint32, 68) + setRnd(input, 4) + var bitmap = From(input) + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + for j, e := bitmap.NextSet(0); e; j, e = bitmap.NextSet(j + 1) { + checksum += j + } + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +// function provided by FlorianUekermann +func good(set []uint32) (checksum uint) { + for wordIdx, word := range set { + var wordIdx = uint(wordIdx * 64) + for word != 0 { + var bitIdx = uint(trailingZeroes32(word)) + word ^= 1 << bitIdx + var index = wordIdx + bitIdx + checksum += index + } + } + return checksum +} + +func BenchmarkFlorianUekermannIterateManyComp(b *testing.B) { + var input = make([]uint32, 68) + setRnd(input, 4) + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + checksum += good(input) + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +/////// Mid density + +// go test -bench=BenchmarkFlorianUekermannLowDensityIterateMany +func BenchmarkFlorianUekermannLowDensityIterateMany(b *testing.B) { + var input = make([]uint32, 1000000) + var rnd = rand.NewSource(0).(rand.Source64) + for i := 0; i < 50000; i++ { + input[rnd.Uint64()%1000000] = 1 + } + var bitmap = From(input) + buffer := make([]uint, 256) + b.ResetTimer() + var sum = uint(0) + for i := 0; i < b.N; i++ { + j := uint(0) + j, buffer = bitmap.NextSetMany(j, buffer) + for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j, buffer) { + for k := range buffer { + sum += buffer[k] + } + j++ + } + } + if sum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannLowDensityIterateManyReg(b *testing.B) { + var input = make([]uint32, 1000000) + var rnd = rand.NewSource(0).(rand.Source64) + for i := 0; i < 50000; i++ { + input[rnd.Uint64()%1000000] = 1 + } + var bitmap = From(input) + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + for j, e := bitmap.NextSet(0); e; j, e = bitmap.NextSet(j + 1) { + checksum += j + } + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannLowDensityIterateManyComp(b *testing.B) { + var input = make([]uint32, 1000000) + var rnd = rand.NewSource(0).(rand.Source64) + for i := 0; i < 50000; i++ { + input[rnd.Uint64()%1000000] = 1 + } + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + checksum += good(input) + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +/////// Mid density + +// go test -bench=BenchmarkFlorianUekermannMidDensityIterateMany +func BenchmarkFlorianUekermannMidDensityIterateMany(b *testing.B) { + var input = make([]uint32, 1000000) + var rndSource = rand.NewSource(0) + var rnd = rand.New(rndSource) + for i := 0; i < 3000000; i++ { + input[rnd.Uint32()%1000000] |= uint32(1) << (rnd.Uint32() % 32) + } + var bitmap = From(input) + buffer := make([]uint, 256) + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + j := uint(0) + j, buffer = bitmap.NextSetMany(j, buffer) + for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j, buffer) { + for k := range buffer { + sum += buffer[k] + } + j++ + } + } + + if sum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannMidDensityIterateManyReg(b *testing.B) { + var input = make([]uint32, 1000000) + var rndSource = rand.NewSource(0) + var rnd = rand.New(rndSource) + for i := 0; i < 3000000; i++ { + input[rnd.Uint32()%1000000] |= uint32(1) << (rnd.Uint32() % 32) + } + var bitmap = From(input) + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + for j, e := bitmap.NextSet(0); e; j, e = bitmap.NextSet(j + 1) { + checksum += j + } + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannMidDensityIterateManyComp(b *testing.B) { + var input = make([]uint32, 1000000) + var rndSource = rand.NewSource(0) + var rnd = rand.New(rndSource) + for i := 0; i < 3000000; i++ { + input[rnd.Uint32()%1000000] |= uint32(1) << (rnd.Uint32() % 32) + } + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + checksum += good(input) + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +////////// High density + +func BenchmarkFlorianUekermannMidStrongDensityIterateMany(b *testing.B) { + var input = make([]uint32, 1000000) + var rndSource = rand.NewSource(0) + var rnd = rand.New(rndSource) + for i := 0; i < 20000000; i++ { + input[rnd.Uint32()%1000000] |= uint32(1) << (rnd.Uint32() % 64) + } + var bitmap = From(input) + buffer := make([]uint, 256) + b.ResetTimer() + sum := uint(0) + for i := 0; i < b.N; i++ { + j := uint(0) + j, buffer = bitmap.NextSetMany(j, buffer) + for ; len(buffer) > 0; j, buffer = bitmap.NextSetMany(j, buffer) { + for k := range buffer { + sum += buffer[k] + } + j++ + } + } + + if sum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannMidStrongDensityIterateManyReg(b *testing.B) { + var input = make([]uint32, 1000000) + var rndSource = rand.NewSource(0) + var rnd = rand.New(rndSource) + for i := 0; i < 20000000; i++ { + input[rnd.Uint32()%1000000] |= uint32(1) << (rnd.Uint32() % 32) + } + var bitmap = From(input) + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + for j, e := bitmap.NextSet(0); e; j, e = bitmap.NextSet(j + 1) { + checksum += j + } + } + if checksum == 0 { // added just to fool ineffassign + return + } +} + +func BenchmarkFlorianUekermannMidStrongDensityIterateManyComp(b *testing.B) { + var input = make([]uint32, 1000000) + var rndSource = rand.NewSource(0) + var rnd = rand.New(rndSource) + for i := 0; i < 20000000; i++ { + input[rnd.Uint32()%1000000] |= uint32(1) << (rnd.Uint32() % 32) + } + b.ResetTimer() + var checksum = uint(0) + for i := 0; i < b.N; i++ { + checksum += good(input) + } + if checksum == 0 { // added just to fool ineffassign + return + } +} diff --git a/common/serialize/bitset/bitset32_test.go b/common/serialize/bitset/bitset32_test.go new file mode 100644 index 000000000..d694f1d6c --- /dev/null +++ b/common/serialize/bitset/bitset32_test.go @@ -0,0 +1,2011 @@ +// Copyright 2014 Will Fitzgerald. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// This file tests bit sets + +package bitset32 + +import ( + "bytes" + "compress/gzip" + "encoding/base32" + "errors" + "fmt" + "io" + "math" + "strconv" + "testing" +) + +func TestStringer(t *testing.T) { + v := New(0) + for i := uint(0); i < 10; i++ { + v.Set(i) + } + if v.String() != "{0,1,2,3,4,5,6,7,8,9}" { + t.Error("bad string output") + } +} + +func TestStringLong(t *testing.T) { + v := New(0) + for i := uint(0); i < 262145; i++ { + v.Set(i) + } + str := v.String() + if len(str) != 1723903 { + t.Error("Unexpected string length: ", len(str)) + } +} + +func TestEmptyBitSet(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Error("A zero-length BitSet32 should be fine") + } + }() + b := New(0) + if b.Len() != 0 { + t.Errorf("Empty set should have capacity 0, not %d", b.Len()) + } +} + +func TestZeroValueBitSet(t *testing.T) { + defer func() { + if r := recover(); r != nil { + t.Error("A zero-length BitSet32 should be fine") + } + }() + var b BitSet32 + if b.Len() != 0 { + t.Errorf("Empty set should have capacity 0, not %d", b.Len()) + } +} + +func TestBitSetNew(t *testing.T) { + v := New(16) + if v.Test(0) { + t.Errorf("Unable to make a bit set and read its 0th value.") + } +} + +func TestBitSetHuge(t *testing.T) { + v := New(uint(math.MaxUint32)) + if v.Test(0) { + t.Errorf("Unable to make a huge bit set and read its 0th value.") + } +} + +func TestLen(t *testing.T) { + v := New(1000) + if v.Len() != 1000 { + t.Errorf("Len should be 1000, but is %d.", v.Len()) + } +} + +func TestLenIsNumberOfBitsNotBytes(t *testing.T) { + var b BitSet32 + if b.Len() != 0 { + t.Errorf("empty BitSet32 should have Len 0, got %v", b.Len()) + } + + b.Set(0) + if b.Len() != 1 { + t.Errorf("BitSet32 with first bit set should have Len 1, got %v", b.Len()) + } + + b.Set(8) + if b.Len() != 9 { + t.Errorf("BitSet32 with 0th and 8th bit set should have Len 9, got %v", b.Len()) + } + + b.Set(1) + if b.Len() != 9 { + t.Errorf("BitSet32 with 0th, 1st and 8th bit set should have Len 9, got %v", b.Len()) + } +} + +func ExampleBitSet_Len() { + var b BitSet32 + b.Set(8) + fmt.Println("len", b.Len()) + fmt.Println("count", b.Count()) + // Output: + // len 9 + // count 1 +} + +func TestBitSetIsClear(t *testing.T) { + v := New(1000) + for i := uint(0); i < 1000; i++ { + if v.Test(i) { + t.Errorf("Bit %d is set, and it shouldn't be.", i) + } + } +} + +func TestExendOnBoundary(t *testing.T) { + v := New(32) + defer func() { + if r := recover(); r != nil { + t.Error("Border out of index error should not have caused a panic") + } + }() + v.Set(32) +} + +func TestExceedCap(t *testing.T) { + defer func() { + if r := recover(); r == nil { + t.Error("Set to capacity should have caused a panic") + } + }() + NumHosts := uint(32768) + bmp := New(NumHosts) + bmp.ClearAll() + d := Cap() + bmp.Set(d) + +} + +func TestExpand(t *testing.T) { + v := New(0) + defer func() { + if r := recover(); r != nil { + t.Error("Expansion should not have caused a panic") + } + }() + for i := uint(0); i < 1000; i++ { + v.Set(i) + } +} + +func TestBitSetAndGet(t *testing.T) { + v := New(1000) + v.Set(100) + if !v.Test(100) { + t.Errorf("Bit %d is clear, and it shouldn't be.", 100) + } +} + +func TestNextClear(t *testing.T) { + v := New(1000) + v.Set(0).Set(1) + next, found := v.NextClear(0) + if !found || next != 2 { + t.Errorf("Found next clear bit as %d, it should have been 2", next) + } + + v = New(1000) + for i := uint(0); i < 66; i++ { + v.Set(i) + } + next, found = v.NextClear(0) + if !found || next != 66 { + t.Errorf("Found next clear bit as %d, it should have been 66", next) + } + + v = New(1000) + for i := uint(0); i < 64; i++ { + v.Set(i) + } + v.Clear(45) + v.Clear(52) + next, found = v.NextClear(10) + if !found || next != 45 { + t.Errorf("Found next clear bit as %d, it should have been 45", next) + } + + v = New(1000) + for i := uint(0); i < 128; i++ { + v.Set(i) + } + v.Clear(73) + v.Clear(99) + next, found = v.NextClear(10) + if !found || next != 73 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + + next, found = v.NextClear(72) + if !found || next != 73 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + next, found = v.NextClear(73) + if !found || next != 73 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + next, found = v.NextClear(74) + if !found || next != 99 { + t.Errorf("Found next clear bit as %d, it should have been 73", next) + } + + v = New(128) + next, found = v.NextClear(0) + if !found || next != 0 { + t.Errorf("Found next clear bit as %d, it should have been 0", next) + } + + for i := uint(0); i < 128; i++ { + v.Set(i) + } + _, found = v.NextClear(0) + if found { + t.Errorf("There are not clear bits") + } + + b := new(BitSet32) + c, d := b.NextClear(1) + if c != 0 || d { + t.Error("Unexpected values") + return + } + + v = New(100) + for i := uint(0); i != 100; i++ { + v.Set(i) + } + next, found = v.NextClear(0) + if found || next != 0 { + t.Errorf("Found next clear bit as %d, it should have return (0, false)", next) + + } +} + +func TestIterate(t *testing.T) { + v := New(10000) + v.Set(0) + v.Set(1) + v.Set(2) + data := make([]uint, 3) + c := 0 + for i, e := v.NextSet(0); e; i, e = v.NextSet(i + 1) { + data[c] = i + c++ + } + if data[0] != 0 { + t.Errorf("bug 0") + } + if data[1] != 1 { + t.Errorf("bug 1") + } + if data[2] != 2 { + t.Errorf("bug 2") + } + v.Set(10) + v.Set(2000) + data = make([]uint, 5) + c = 0 + for i, e := v.NextSet(0); e; i, e = v.NextSet(i + 1) { + data[c] = i + c++ + } + if data[0] != 0 { + t.Errorf("bug 0") + } + if data[1] != 1 { + t.Errorf("bug 1") + } + if data[2] != 2 { + t.Errorf("bug 2") + } + if data[3] != 10 { + t.Errorf("bug 3") + } + if data[4] != 2000 { + t.Errorf("bug 4") + } + +} + +func TestSetTo(t *testing.T) { + v := New(1000) + v.SetTo(100, true) + if !v.Test(100) { + t.Errorf("Bit %d is clear, and it shouldn't be.", 100) + } + v.SetTo(100, false) + if v.Test(100) { + t.Errorf("Bit %d is set, and it shouldn't be.", 100) + } +} + +func TestChain(t *testing.T) { + if !New(1000).Set(100).Set(99).Clear(99).Test(100) { + t.Errorf("Bit %d is clear, and it shouldn't be.", 100) + } +} + +func TestOutOfBoundsLong(t *testing.T) { + v := New(64) + defer func() { + if r := recover(); r != nil { + t.Error("Long distance out of index error should not have caused a panic") + } + }() + v.Set(1000) +} + +func TestOutOfBoundsClose(t *testing.T) { + v := New(65) + defer func() { + if r := recover(); r != nil { + t.Error("Local out of index error should not have caused a panic") + } + }() + v.Set(66) +} + +func TestCount(t *testing.T) { + tot := uint(64*4 + 11) // just some multi unit64 number + v := New(tot) + checkLast := true + for i := uint(0); i < tot; i++ { + sz := uint(v.Count()) + if sz != i { + t.Errorf("Count reported as %d, but it should be %d", sz, i) + checkLast = false + break + } + v.Set(i) + } + if checkLast { + sz := uint(v.Count()) + if sz != tot { + t.Errorf("After all bits set, size reported as %d, but it should be %d", sz, tot) + } + } +} + +// test setting every 3rd bit, just in case something odd is happening +func TestCount2(t *testing.T) { + tot := uint(64*4 + 11) // just some multi unit64 number + v := New(tot) + for i := uint(0); i < tot; i += 3 { + sz := uint(v.Count()) + if sz != i/3 { + t.Errorf("Count reported as %d, but it should be %d", sz, i) + break + } + v.Set(i) + } +} + +// nil tests +func TestNullTest(t *testing.T) { + var v *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Checking bit of null reference should have caused a panic") + } + }() + v.Test(66) +} + +func TestNullSet(t *testing.T) { + var v *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Setting bit of null reference should have caused a panic") + } + }() + v.Set(66) +} + +func TestNullClear(t *testing.T) { + var v *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Clearning bit of null reference should have caused a panic") + } + }() + v.Clear(66) +} + +func TestNullCount(t *testing.T) { + var v *BitSet32 + defer func() { + if r := recover(); r != nil { + t.Error("Counting null reference should not have caused a panic") + } + }() + cnt := v.Count() + if cnt != 0 { + t.Errorf("Count reported as %d, but it should be 0", cnt) + } +} + +func TestPanicDifferenceBNil(t *testing.T) { + var b *BitSet32 + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.Difference(compare) +} + +func TestPanicDifferenceCompareNil(t *testing.T) { + var compare *BitSet32 + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.Difference(compare) +} + +func TestPanicUnionBNil(t *testing.T) { + var b *BitSet32 + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.Union(compare) +} + +func TestPanicUnionCompareNil(t *testing.T) { + var compare *BitSet32 + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.Union(compare) +} + +func TestPanicIntersectionBNil(t *testing.T) { + var b *BitSet32 + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.Intersection(compare) +} + +func TestPanicIntersectionCompareNil(t *testing.T) { + var compare *BitSet32 + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.Intersection(compare) +} + +func TestPanicSymmetricDifferenceBNil(t *testing.T) { + var b *BitSet32 + var compare = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil First should should have caused a panic") + } + }() + b.SymmetricDifference(compare) +} + +func TestPanicSymmetricDifferenceCompareNil(t *testing.T) { + var compare *BitSet32 + var b = New(10) + defer func() { + if r := recover(); r == nil { + t.Error("Nil Second should should have caused a panic") + } + }() + b.SymmetricDifference(compare) +} + +func TestPanicComplementBNil(t *testing.T) { + var b *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.Complement() +} + +func TestPanicAnytBNil(t *testing.T) { + var b *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.Any() +} + +func TestPanicNonetBNil(t *testing.T) { + var b *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.None() +} + +func TestPanicAlltBNil(t *testing.T) { + var b *BitSet32 + defer func() { + if r := recover(); r == nil { + t.Error("Nil should should have caused a panic") + } + }() + b.All() +} + +func TestAll(t *testing.T) { + v := New(0) + if !v.All() { + t.Error("Empty sets should return true on All()") + } + v = New(2) + v.SetTo(0, true) + v.SetTo(1, true) + if !v.All() { + t.Error("Non-empty sets with all bits set should return true on All()") + } + v = New(2) + if v.All() { + t.Error("Non-empty sets with no bits set should return false on All()") + } + v = New(2) + v.SetTo(0, true) + if v.All() { + t.Error("Non-empty sets with some bits set should return false on All()") + } +} + +func TestShrink(t *testing.T) { + bs := New(10) + bs.Set(0) + bs.Shrink(63) + if !bs.Test(0) { + t.Error("0 should be set") + return + } + b := New(0) + + b.Set(0) + b.Set(1) + b.Set(2) + b.Set(3) + b.Set(64) + b.Compact() + if !b.Test(0) { + t.Error("0 should be set") + return + } + if !b.Test(1) { + t.Error("1 should be set") + return + } + if !b.Test(2) { + t.Error("2 should be set") + return + } + if !b.Test(3) { + t.Error("3 should be set") + return + } + if !b.Test(64) { + t.Error("64 should be set") + return + } + + b.Shrink(2) + if !b.Test(0) { + t.Error("0 should be set") + return + } + if !b.Test(1) { + t.Error("1 should be set") + return + } + if !b.Test(2) { + t.Error("2 should be set") + return + } + if b.Test(3) { + t.Error("3 should not be set") + return + } + if b.Test(64) { + t.Error("64 should not be set") + return + } + + b.Set(24) + b.Shrink(100) + if !b.Test(24) { + t.Error("24 should be set") + return + } + + b.Set(127) + b.Set(128) + b.Set(129) + b.Compact() + if !b.Test(127) { + t.Error("127 should be set") + return + } + if !b.Test(128) { + t.Error("128 should be set") + return + } + if !b.Test(129) { + t.Error("129 be set") + return + } + + b.Shrink(128) + if !b.Test(127) { + t.Error("127 should be set") + return + } + if !b.Test(128) { + t.Error("128 should be set") + return + } + if b.Test(129) { + t.Error("129 should not be set") + return + } + + b.Set(129) + b.Shrink(129) + if !b.Test(129) { + t.Error("129 should be set") + return + } + + b.Set(1000) + b.Set(2000) + b.Set(3000) + b.Shrink(3000) + if len(b.set) != 3000/32+1 { + t.Error("Wrong length of BitSet32.set") + return + } + if !b.Test(3000) { + t.Error("3000 should be set") + return + } + + b.Shrink(2000) + if len(b.set) != 2000/32+1 { + t.Error("Wrong length of BitSet32.set") + return + } + if b.Test(3000) { + t.Error("3000 should not be set") + return + } + if !b.Test(2000) { + t.Error("2000 should be set") + return + } + if !b.Test(1000) { + t.Error("1000 should be set") + return + } + if !b.Test(24) { + t.Error("24 should be set") + return + } + + b = New(110) + b.Set(80) + b.Shrink(70) + for _, word := range b.set { + if word != 0 { + t.Error("word should be 0", word) + } + } +} + +func TestInsertAtWithSet(t *testing.T) { + b := New(0) + b.Set(0) + b.Set(1) + b.Set(63) + b.Set(64) + b.Set(65) + + b.InsertAt(3) + if !b.Test(0) { + t.Error("0 should be set") + return + } + if !b.Test(1) { + t.Error("1 should be set") + return + } + if b.Test(3) { + t.Error("3 should not be set") + return + } + if !b.Test(64) { + t.Error("64 should be set") + return + } + if !b.Test(65) { + t.Error("65 should be set") + return + } + if !b.Test(66) { + t.Error("66 should be set") + return + } + +} + +func TestInsertAt(t *testing.T) { + type testCase struct { + input []string + insertIdx uint + expected []string + } + + testCases := []testCase{ + { + input: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + }, + insertIdx: uint(62), + expected: []string{ + "11111111111111111111111111111111", + "10111111111111111111111111111111", + "00000000000000000000000000000001", + }, + }, + { + input: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + }, + insertIdx: uint(63), + expected: []string{ + "11111111111111111111111111111111", + "01111111111111111111111111111111", + "00000000000000000000000000000001", + }, + }, + { + input: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + }, + insertIdx: uint(0), + expected: []string{ + "11111111111111111111111111111110", + "11111111111111111111111111111111", + "00000000000000000000000000000001", + }, + }, + { + input: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + }, + insertIdx: uint(70), + expected: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111110111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "00000000000000000000000000000001", + }, + }, + { + input: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111111110000", + "11111111111111111111111111111111", + }, + insertIdx: uint(70), + expected: []string{ + "11111111111111111111111111111111", + "11111111111111111111111111111111", + "11111111111111111111111110111111", + "11111111111111111111111111111111", + "11111111111111111111111111100001", + "11111111111111111111111111111111", + "00000000000000000000000000000001", + }, + }, + { + input: []string{ + "11111111111111111111111111110000", + "11111111111111111111111111111111", + }, + insertIdx: uint(10), + expected: []string{ + "11111111111111111111101111110000", + "11111111111111111111111111111111", + "00000000000000000000000000000001", + }, + }, + } + + for _, tc := range testCases { + var input []uint32 + for _, inputElement := range tc.input { + parsed, _ := strconv.ParseUint(inputElement, 2, 32) + input = append(input, uint32(parsed)) + } + + var expected []uint32 + for _, expectedElement := range tc.expected { + parsed, _ := strconv.ParseUint(expectedElement, 2, 32) + expected = append(expected, uint32(parsed)) + } + + b := From(input) + b.InsertAt(tc.insertIdx) + if len(b.set) != len(expected) { + t.Error("Length of sets should be equal") + return + } + for i := range b.set { + if b.set[i] != expected[i] { + t.Error("Unexpected results found in set") + return + } + } + } +} + +func TestNone(t *testing.T) { + v := New(0) + if !v.None() { + t.Error("Empty sets should return true on None()") + } + v = New(2) + v.SetTo(0, true) + v.SetTo(1, true) + if v.None() { + t.Error("Non-empty sets with all bits set should return false on None()") + } + v = New(2) + if !v.None() { + t.Error("Non-empty sets with no bits set should return true on None()") + } + v = New(2) + v.SetTo(0, true) + if v.None() { + t.Error("Non-empty sets with some bits set should return false on None()") + } + v = new(BitSet32) + if !v.None() { + t.Error("Empty sets should return true on None()") + } +} + +func TestEqual(t *testing.T) { + a := New(100) + b := New(99) + c := New(100) + if a.Equal(b) { + t.Error("Sets of different sizes should be not be equal") + } + if !a.Equal(c) { + t.Error("Two empty sets of the same size should be equal") + } + a.Set(99) + c.Set(0) + if a.Equal(c) { + t.Error("Two sets with differences should not be equal") + } + c.Set(99) + a.Set(0) + if !a.Equal(c) { + t.Error("Two sets with the same bits set should be equal") + } + if a.Equal(nil) { + t.Error("The sets should be different") + } + a = New(0) + b = New(0) + if !a.Equal(b) { + t.Error("Two empty set should be equal") + } + var x *BitSet32 + var y *BitSet32 + z := New(0) + if !x.Equal(y) { + t.Error("Two nil bitsets should be equal") + } + if x.Equal(z) { + t.Error("Nil receiver BitSet32 should not be equal to non-nil BitSet32") + } +} + +func TestUnion(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.UnionCardinality(b) != 200 { + t.Errorf("Union should have 200 bits set, but had %d", a.UnionCardinality(b)) + } + if a.UnionCardinality(b) != b.UnionCardinality(a) { + t.Errorf("Union should be symmetric") + } + c := a.Union(b) + d := b.Union(a) + if c.Count() != 200 { + t.Errorf("Union should have 200 bits set, but had %d", c.Count()) + } + if !c.Equal(d) { + t.Errorf("Union should be symmetric") + } +} + +func TestInPlaceUnion(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceUnion(b) + d := b.Clone() + d.InPlaceUnion(a) + if c.Count() != 200 { + t.Errorf("Union should have 200 bits set, but had %d", c.Count()) + } + if d.Count() != 200 { + t.Errorf("Union should have 200 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("Union should be symmetric") + } +} + +func TestIntersection(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1).Set(i) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.IntersectionCardinality(b) != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", a.IntersectionCardinality(b)) + } + if a.IntersectionCardinality(b) != b.IntersectionCardinality(a) { + t.Errorf("Intersection should be symmetric") + } + c := a.Intersection(b) + d := b.Intersection(a) + if c.Count() != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", c.Count()) + } + if !c.Equal(d) { + t.Errorf("Intersection should be symmetric") + } +} + +func TestInplaceIntersection(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1).Set(i) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceIntersection(b) + d := b.Clone() + d.InPlaceIntersection(a) + if c.Count() != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", c.Count()) + } + if d.Count() != 50 { + t.Errorf("Intersection should have 50 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("Intersection should be symmetric") + } +} + +func TestDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.DifferenceCardinality(b) != 50 { + t.Errorf("a-b Difference should have 50 bits set, but had %d", a.DifferenceCardinality(b)) + } + if b.DifferenceCardinality(a) != 150 { + t.Errorf("b-a Difference should have 150 bits set, but had %d", b.DifferenceCardinality(a)) + } + + c := a.Difference(b) + d := b.Difference(a) + if c.Count() != 50 { + t.Errorf("a-b Difference should have 50 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b-a Difference should have 150 bits set, but had %d", d.Count()) + } + if c.Equal(d) { + t.Errorf("Difference, here, should not be symmetric") + } +} + +func TestInPlaceDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) + b.Set(i - 1) + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceDifference(b) + d := b.Clone() + d.InPlaceDifference(a) + if c.Count() != 50 { + t.Errorf("a-b Difference should have 50 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b-a Difference should have 150 bits set, but had %d", d.Count()) + } + if c.Equal(d) { + t.Errorf("Difference, here, should not be symmetric") + } +} + +func TestSymmetricDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) // 01010101010 ... 0000000 + b.Set(i - 1).Set(i) // 11111111111111111000000 + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + if a.SymmetricDifferenceCardinality(b) != 150 { + t.Errorf("a^b Difference should have 150 bits set, but had %d", a.SymmetricDifferenceCardinality(b)) + } + if b.SymmetricDifferenceCardinality(a) != 150 { + t.Errorf("b^a Difference should have 150 bits set, but had %d", b.SymmetricDifferenceCardinality(a)) + } + + c := a.SymmetricDifference(b) + d := b.SymmetricDifference(a) + if c.Count() != 150 { + t.Errorf("a^b Difference should have 150 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b^a Difference should have 150 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("SymmetricDifference should be symmetric") + } +} + +func TestInPlaceSymmetricDifference(t *testing.T) { + a := New(100) + b := New(200) + for i := uint(1); i < 100; i += 2 { + a.Set(i) // 01010101010 ... 0000000 + b.Set(i - 1).Set(i) // 11111111111111111000000 + } + for i := uint(100); i < 200; i++ { + b.Set(i) + } + c := a.Clone() + c.InPlaceSymmetricDifference(b) + d := b.Clone() + d.InPlaceSymmetricDifference(a) + if c.Count() != 150 { + t.Errorf("a^b Difference should have 150 bits set, but had %d", c.Count()) + } + if d.Count() != 150 { + t.Errorf("b^a Difference should have 150 bits set, but had %d", d.Count()) + } + if !c.Equal(d) { + t.Errorf("SymmetricDifference should be symmetric") + } +} + +func TestComplement(t *testing.T) { + a := New(50) + b := a.Complement() + if b.Count() != 50 { + t.Errorf("Complement failed, size should be 50, but was %d", b.Count()) + } + a = New(50) + a.Set(10).Set(20).Set(42) + b = a.Complement() + if b.Count() != 47 { + t.Errorf("Complement failed, size should be 47, but was %d", b.Count()) + } +} + +func TestIsSuperSet(t *testing.T) { + a := New(500) + b := New(300) + c := New(200) + + // Setup bitsets + // a and b overlap + // only c is (strict) super set + for i := uint(0); i < 100; i++ { + a.Set(i) + } + for i := uint(50); i < 150; i++ { + b.Set(i) + } + for i := uint(0); i < 200; i++ { + c.Set(i) + } + + if a.IsSuperSet(b) { + t.Errorf("IsSuperSet fails") + } + if a.IsSuperSet(c) { + t.Errorf("IsSuperSet fails") + } + if b.IsSuperSet(a) { + t.Errorf("IsSuperSet fails") + } + if b.IsSuperSet(c) { + t.Errorf("IsSuperSet fails") + } + if !c.IsSuperSet(a) { + t.Errorf("IsSuperSet fails") + } + if !c.IsSuperSet(b) { + t.Errorf("IsSuperSet fails") + } + if a.IsStrictSuperSet(b) { + t.Errorf("IsStrictSuperSet fails") + } + if a.IsStrictSuperSet(c) { + t.Errorf("IsStrictSuperSet fails") + } + if b.IsStrictSuperSet(a) { + t.Errorf("IsStrictSuperSet fails") + } + if b.IsStrictSuperSet(c) { + t.Errorf("IsStrictSuperSet fails") + } + if !c.IsStrictSuperSet(a) { + t.Errorf("IsStrictSuperSet fails") + } + if !c.IsStrictSuperSet(b) { + t.Errorf("IsStrictSuperSet fails") + } +} + +func TestDumpAsBits(t *testing.T) { + a := New(10).Set(10) + astr := "0000000000000000000000000000000000000000000000000000010000000000." + if a.DumpAsBits() != astr { + t.Errorf("DumpAsBits failed, output should be \"%s\" but was \"%s\"", astr, a.DumpAsBits()) + } + var b BitSet32 // zero value (b.set == nil) + bstr := "." + if b.DumpAsBits() != bstr { + t.Errorf("DumpAsBits failed, output should be \"%s\" but was \"%s\"", bstr, b.DumpAsBits()) + } +} + +// func TestMarshalUnmarshalBinary(t *testing.T) { +// a := New(1010).Set(10).Set(1001) +// b := new(BitSet32) + +// copyBinary(t, a, b) + +// // BitSets must be equal after marshalling and unmarshalling +// if !a.Equal(b) { +// t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) +// return +// } + +// aSetBit := uint(128) +// a = New(256).Set(aSetBit) +// aExpectedMarshaledSize := 8 /* length: uint32 */ + 2*4*8 /* set : [8]uint32 */ +// aMarshaled, err := a.MarshalBinary() +// testsize := a.BinaryStorageSize() +// fmt.Println(testsize) +// if err != nil || aExpectedMarshaledSize != len(aMarshaled) || aExpectedMarshaledSize != a.BinaryStorageSize() { +// t.Error("MarshalBinary failed to produce expected (", aExpectedMarshaledSize, ") number of bytes") +// return +// } + +// shiftAmount := uint(72) +// // https://github.com/bits-and-blooms/BitSet32/issues/114 +// for i := uint(0); i < shiftAmount; i++ { +// a.DeleteAt(0) +// } + +// aExpectedMarshaledSize = 8 /* length: uint32 */ + 2*3*8 /* set : [6]uint32 */ +// aMarshaled, err = a.MarshalBinary() +// if err != nil || aExpectedMarshaledSize != len(aMarshaled) || aExpectedMarshaledSize != a.BinaryStorageSize() { +// t.Error("MarshalBinary failed to produce expected (", aExpectedMarshaledSize, ") number of bytes") +// return +// } + +// copyBinary(t, a, b) + +// if b.Len() != 256-shiftAmount || !b.Test(aSetBit-shiftAmount) { +// t.Error("Shifted BitSet32 is not copied correctly") +// } +// } + +// func TestMarshalUnmarshalBinaryByLittleEndian(t *testing.T) { +// LittleEndian() +// defer func() { +// // Revert when done. +// binaryOrder = binary.BigEndian +// }() +// a := New(1010).Set(10).Set(1001) +// b := new(BitSet32) + +// copyBinary(t, a, b) + +// // BitSets must be equal after marshalling and unmarshalling +// if !a.Equal(b) { +// t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) +// return +// } +// } + +// func copyBinary(t *testing.T, from encoding.BinaryMarshaler, to encoding.BinaryUnmarshaler) { +// data, err := from.MarshalBinary() +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// err = to.UnmarshalBinary(data) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } +// } + +// func TestMarshalUnmarshalJSON(t *testing.T) { +// a := New(1010).Set(10).Set(1001) +// data, err := json.Marshal(a) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// b := new(BitSet32) +// err = json.Unmarshal(data, b) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// // Bitsets must be equal after marshalling and unmarshalling +// if !a.Equal(b) { +// t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) +// return +// } +// } + +// func TestMarshalUnmarshalJSONWithTrailingData(t *testing.T) { +// a := New(1010).Set(10).Set(1001) +// data, err := json.Marshal(a) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// // appending some noise +// data = data[:len(data)-3] // remove " +// data = append(data, []byte(`AAAAAAAAAA"`)...) + +// b := new(BitSet32) +// err = json.Unmarshal(data, b) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// // Bitsets must be equal after marshalling and unmarshalling +// // Do not over-reading when unmarshalling +// if !a.Equal(b) { +// t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) +// return +// } +// } + +// func TestMarshalUnmarshalJSONByStdEncoding(t *testing.T) { +// Base64StdEncoding() +// a := New(1010).Set(10).Set(1001) +// data, err := json.Marshal(a) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// b := new(BitSet32) +// err = json.Unmarshal(data, b) +// if err != nil { +// t.Errorf(err.Error()) +// return +// } + +// // Bitsets must be equal after marshalling and unmarshalling +// if !a.Equal(b) { +// t.Error("Bitsets are not equal:\n\t", a.DumpAsBits(), "\n\t", b.DumpAsBits()) +// return +// } +// } + +func TestSafeSet(t *testing.T) { + b := new(BitSet32) + c := b.safeSet() + outType := fmt.Sprintf("%T", c) + expType := "[]uint32" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } + if len(c) != 0 { + t.Error("The slice should be empty") + return + } +} + +func TestSetBitsetFrom(t *testing.T) { + u := []uint32{2, 3, 5, 7, 11} + b := new(BitSet32) + b.SetBitsetFrom(u) + outType := fmt.Sprintf("%T", b) + expType := "*bitset32.BitSet32" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } +} + +func TestIssue116(t *testing.T) { + a := []uint32{2, 3, 5, 7, 11} + b := []uint32{2, 3, 5, 7, 11, 0, 1} + bitset1 := FromWithLength(160, a) + bitset2 := FromWithLength(160, b) + if !bitset1.Equal(bitset2) || !bitset2.Equal(bitset1) { + t.Error("Bitsets should be equal irrespective of the underlying capacity") + } +} + +func TestFrom(t *testing.T) { + u := []uint32{2, 3, 5, 7, 11} + b := From(u) + outType := fmt.Sprintf("%T", b) + expType := "*bitset32.BitSet32" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } +} + +func TestBytes(t *testing.T) { + b := new(BitSet32) + c := b.Bytes() + outType := fmt.Sprintf("%T", c) + expType := "[]uint32" + if outType != expType { + t.Error("Expecting type: ", expType, ", gotf:", outType) + return + } + if len(c) != 0 { + t.Error("The slice should be empty") + return + } +} + +func TestCap(t *testing.T) { + c := Cap() + if c <= 0 { + t.Error("The uint capacity should be >= 0") + return + } +} + +func TestWordsNeededLong(t *testing.T) { + i := Cap() + out := wordsNeeded(i) + if out <= 0 { + t.Error("Unexpected value: ", out) + return + } +} + +func TestTestTooLong(t *testing.T) { + b := new(BitSet32) + if b.Test(1) { + t.Error("Unexpected value: true") + return + } +} + +func TestClearTooLong(t *testing.T) { + b := new(BitSet32) + c := b.Clear(1) + if b != c { + t.Error("Unexpected value") + return + } +} + +func TestClearAll(t *testing.T) { + u := []uint32{2, 3, 5, 7, 11} + b := From(u) + c := b.ClearAll() + if c.length != 160 { + t.Error("Unexpected length: ", b.length) + return + } + if c.Test(0) || c.Test(1) || c.Test(2) || c.Test(3) || c.Test(4) || c.Test(5) { + t.Error("All bits should be unset") + return + } +} + +func TestFlip(t *testing.T) { + b := new(BitSet32) + c := b.Flip(11) + if c.length != 12 { + t.Error("Unexpected value: ", c.length) + return + } + d := c.Flip(7) + if d.length != 12 { + t.Error("Unexpected value: ", d.length) + return + } +} + +func TestFlipRange(t *testing.T) { + b := new(BitSet32) + b.Set(1).Set(3).Set(5).Set(7).Set(9).Set(11).Set(13).Set(15) + c := b.FlipRange(4, 25) + if c.length != 25 { + t.Error("Unexpected value: ", c.length) + return + } + d := c.FlipRange(8, 24) + if d.length != 25 { + t.Error("Unexpected value: ", d.length) + return + } + // + for i := uint(0); i < 256; i++ { + for j := uint(0); j <= i; j++ { + bits := New(i) + bits.FlipRange(0, j) + c := bits.Count() + if c != j { + t.Error("Unexpected value: ", c, " expected: ", j) + return + } + } + } +} + +func TestCopy(t *testing.T) { + a := New(10) + if a.Copy(nil) != 0 { + t.Error("No values should be copied") + return + } + a = New(10) + b := New(20) + if a.Copy(b) != 10 { + t.Error("Unexpected value") + return + } +} + +func TestCopyUnaligned(t *testing.T) { + a := New(16) + a.FlipRange(0, 16) + b := New(1) + a.Copy(b) + if b.Count() > b.Len() { + t.Errorf("targets copied set count (%d) should never be larger than target's length (%d)", b.Count(), b.Len()) + } + if !b.Test(0) { + t.Errorf("first bit should still be set in copy: %+v", b) + } + + // Test a more complex scenario with a mix of bits set in the unaligned space to verify no bits are lost. + a = New(32) + a.Set(0).Set(3).Set(4).Set(16).Set(17).Set(29).Set(31) + b = New(19) + a.Copy(b) + + const expectedCount = 5 + if b.Count() != expectedCount { + t.Errorf("targets copied set count: %d, want %d", b.Count(), expectedCount) + } + + if !(b.Test(0) && b.Test(3) && b.Test(4) && b.Test(16) && b.Test(17)) { + t.Errorf("expected set bits are not set: %+v", b) + } +} + +func TestCopyFull(t *testing.T) { + a := New(10) + b := &BitSet32{} + a.CopyFull(b) + if b.length != a.length || len(b.set) != len(a.set) { + t.Error("Expected full length copy") + return + } + for i, v := range a.set { + if v != b.set[i] { + t.Error("Unexpected value") + return + } + } +} + +func TestNextSetError(t *testing.T) { + b := new(BitSet32) + c, d := b.NextSet(1) + if c != 0 || d { + t.Error("Unexpected values") + return + } +} + +func TestDeleteWithBitStrings(t *testing.T) { + type testCase struct { + input []string + deleteIdx uint + expected []string + } + + testCases := []testCase{ + { + input: []string{ + "00000000000000000000000000000001", + "11100000000000000000000000000000", + }, + deleteIdx: uint(63), + expected: []string{ + "00000000000000000000000000000001", + "01100000000000000000000000000000", + }, + }, + { + input: []string{ + "00000000000000000000000000010101", + "10000000000000000000000000000000", + }, + deleteIdx: uint(0), + expected: []string{ + "00000000000000000000000000001010", + "01000000000000000000000000000000", + }, + }, + { + input: []string{ + "00000000000000000000000000111000", + "00000000000000000000000000000000", + }, + deleteIdx: uint(4), + expected: []string{ + "00000000000000000000000000011000", + "00000000000000000000000000000000", + }, + }, + { + input: []string{ + "00000000000000000000000000000001", + "10000000000000000000000000000000", + "00000000000000000000000000000001", + "10100000000000000000000000000000", + }, + deleteIdx: uint(63), + expected: []string{ + "00000000000000000000000000000001", + "10000000000000000000000000000000", + "00000000000000000000000000000000", + "01010000000000000000000000000000", + }, + }, + { + input: []string{ + "00000000000000000000000000000000", + "10000000000000000000000000000000", + "00000000000000000000000000000001", + "10000000000000000000000000000000", + "00000000000000000000000000000001", + "10000000000000000000000000000000", + }, + deleteIdx: uint(64), + expected: []string{ + "00000000000000000000000000000000", + "10000000000000000000000000000000", + "00000000000000000000000000000000", + "11000000000000000000000000000000", + "00000000000000000000000000000000", + "01000000000000000000000000000000", + }, + }, + { + input: []string{ + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + }, + deleteIdx: uint(256), + expected: []string{ + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000001", + "00000000000000000000000000000000", + "00000000000000000000000000000000", + }, + }, + } + + for _, tc := range testCases { + var input []uint32 + for _, inputElement := range tc.input { + parsed, _ := strconv.ParseUint(inputElement, 2, 32) + input = append(input, uint32(parsed)) + } + + var expected []uint32 + for _, expectedElement := range tc.expected { + parsed, _ := strconv.ParseUint(expectedElement, 2, 32) + expected = append(expected, uint32(parsed)) + } + + b := From(input) + b.DeleteAt(tc.deleteIdx) + if len(b.set) != len(expected) { + t.Errorf("Length of sets expected to be %d, but was %d", len(expected), len(b.set)) + return + } + for i := range b.set { + if b.set[i] != expected[i] { + t.Errorf("Unexpected output\nExpected: %b\nGot: %b", expected[i], b.set[i]) + return + } + } + } +} + +func TestDeleteWithBitSetInstance(t *testing.T) { + length := uint(256) + BitSet32 := New(length) + + // the indexes that get set in the bit set + indexesToSet := []uint{0, 1, 126, 127, 128, 129, 170, 171, 200, 201, 202, 203, 255} + + // the position we delete from the BitSet32 + deleteAt := uint(127) + + // the indexes that we expect to be set after the delete + expectedToBeSet := []uint{0, 1, 126, 127, 128, 169, 170, 199, 200, 201, 202, 254} + + expected := make(map[uint]struct{}) + for _, index := range expectedToBeSet { + expected[index] = struct{}{} + } + + for _, index := range indexesToSet { + BitSet32.Set(index) + } + + BitSet32.DeleteAt(deleteAt) + + for i := uint(0); i < length; i++ { + if _, ok := expected[i]; ok { + if !BitSet32.Test(i) { + t.Errorf("Expected index %d to be set, but wasn't", i) + } + } else { + if BitSet32.Test(i) { + t.Errorf("Expected index %d to not be set, but was", i) + } + } + + } +} + +// //////////////////////////////////////////////////////////////////////TODO +func TestWriteTo(t *testing.T) { + const length = 9585 + const oneEvery = 97 + addBuf := []byte(`12345678`) + bs := New(length) + // Add some bits + for i := uint(0); i < length; i += oneEvery { + bs = bs.Set(i) + } + + var buf bytes.Buffer + n, err := bs.WriteTo(&buf) + if err != nil { + t.Fatal(err) + } + wantSz := buf.Len() // Size of the serialized data in bytes. + if n != int64(wantSz) { + t.Errorf("want write size to be %d, got %d", wantSz, n) + } + buf.Write(addBuf) // Add additional data on stream. + // Generate test input for regression tests: + if false { + gzout := bytes.NewBuffer(nil) + gz, err := gzip.NewWriterLevel(gzout, 9) + if err != nil { + t.Fatal(err) + } + gz.Write(buf.Bytes()) + gz.Close() + t.Log("Encoded:", base32.StdEncoding.EncodeToString(gzout.Bytes())) + } + + // Read back. + bs = New(length) + n, err = bs.ReadFrom(&buf) + if err != nil { + t.Fatal(err) + } + if n != int64(wantSz) { + t.Errorf("want read size to be %d, got %d", wantSz, n) + } + // Check bits + for i := uint(0); i < length; i += oneEvery { + if !bs.Test(i) { + t.Errorf("bit %d was not set", i) + } + } + + more, err := io.ReadAll(&buf) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(more, addBuf) { + t.Fatalf("extra mismatch. got %v, want %v", more, addBuf) + } +} + +type inCompleteRetBufReader struct { + returnEvery int64 + reader io.Reader + offset int64 +} + +func (ir *inCompleteRetBufReader) Read(b []byte) (n int, err error) { + if ir.returnEvery > 0 { + maxRead := ir.returnEvery - (ir.offset % ir.returnEvery) + if len(b) > int(maxRead) { + b = b[:maxRead] + } + } + n, err = ir.reader.Read(b) + ir.offset += int64(n) + return +} + +// TODO: BUGFIX +func TestReadFrom(t *testing.T) { + addBuf := []byte(`12345678`) // Bytes after stream + tests := []struct { + length uint + oneEvery uint + input string // base64+gzipped + wantErr error + returnEvery int64 + }{ + { + length: 9585, + oneEvery: 97, + input: "D6FQQAAAAAAAAAX7MIAAHVKCAYDAMRQGARQEEYVTECYTTEGYAJEGYBJEWYBRFOYBYEODKHUP6GRYCNA2A6R4NE3ARWQZDMMJVGM3SBJAAAAP774SK6XYRQAEAAAA====", + returnEvery: 127, + }, + { + length: 1337, + oneEvery: 42, + input: "D6FQQAAAAAAAAAX7MIAAGVSLAYDAMRQGAYLAMBQBAYDAOBQYQAF4DAADKQEAMBFAIIYIBUYANKMWS2DENRRGUZTOAEEAAAH775L45UAPXAAAAAA=", + }, + { + length: 1337, // Truncated input. + oneEvery: 42, + input: "D6FQQAAAAAAAAAX7MIAAGVSLAYDAMRQGAYLAMBQBAYDAOBQYQAF4DAADKQEAMBFAIIYAAN6A2DENRRGUZTOAEEAAAD777PSLA4MGAAAAAA======", + wantErr: io.ErrUnexpectedEOF, + }, + { + length: 1337, // Empty input. + oneEvery: 42, + input: "D6FQQAAAAAAAAAX7AEAAB777AAAAAAAAAAAAA===", + wantErr: io.ErrUnexpectedEOF, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprint(i), func(t *testing.T) { + fatalErr := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + var buf bytes.Buffer + b, err := base32.StdEncoding.DecodeString(test.input) + fatalErr(err) + gz, err := gzip.NewReader(bytes.NewBuffer(b)) + fatalErr(err) + _, err = io.Copy(&buf, gz) + fatalErr(err) + fatalErr(gz.Close()) + + bs := New(test.length) + _, err = bs.ReadFrom(&inCompleteRetBufReader{returnEvery: test.returnEvery, reader: &buf}) + if err != nil { + if errors.Is(err, test.wantErr) { + // Correct, nothing more we can test. + return + } + t.Fatalf("did not get expected error %v, got %v", test.wantErr, err) + } else { + if test.wantErr != nil { + t.Fatalf("did not get expected error %v", test.wantErr) + } + } + fatalErr(err) + + // Test if correct bits are set. + for i := uint(0); i < test.length; i++ { + want := i%test.oneEvery == 0 + got := bs.Test(i) + if want != got { + t.Errorf("bit %d was %v, should be %v", i, got, want) + } + } + more, err := io.ReadAll(&buf) + fatalErr(err) + if !bytes.Equal(more, addBuf) { + t.Errorf("extra mismatch. got %v, want %v", more, addBuf) + } + }) + } +} + +func TestMaxConsecutiveOne(t *testing.T) { + usecase := []struct { + In []uint32 + Out uint + }{ + { + []uint32{^uint32(0)}, + 32, + }, + { + []uint32{^uint32(0), uint32(1)}, + 33, + }, + { + []uint32{^uint32(0), uint32(0)}, + 32, + }, + { + []uint32{^uint32(0) ^ uint32(1<<10), uint32(0)}, + 21, + }, + { + []uint32{^uint32(0) ^ uint32(1<<10), ^uint32(0)}, + 53, + }, + { + []uint32{^uint32(0) ^ uint32(1<<10), ^uint32(0) ^ uint32(1)}, + 31, + }, + } + for _, uc := range usecase { + bs := From(uc.In) + out := bs.MaxConsecutiveOne(0, bs.length) + if out != uc.Out { + t.Logf("input: %v, want: %v, but got: %v", uc.In, uc.Out, out) + } + } +} + +func TestMaxConsecutiveZero(t *testing.T) { + usecase := []struct { + In []uint32 + Out uint + }{ + { + []uint32{uint32(0)}, + 32, + }, + { + []uint32{uint32(0), ^uint32(0) ^ 1}, + 33, + }, + { + []uint32{uint32(0), ^uint32(0)}, + 32, + }, + { + []uint32{uint32(0) | uint32(1<<10), ^uint32(0)}, + 21, + }, + { + []uint32{uint32(0) | uint32(1<<10), uint32(0)}, + 53, + }, + { + []uint32{uint32(0) | uint32(1<<10), uint32(1)}, + 31, + }, + } + for _, uc := range usecase { + bs := From(uc.In) + out := bs.MaxConsecutiveZero(0, bs.length) + if out != uc.Out { + t.Logf("input: %v, want: %v, but got: %v", uc.In, uc.Out, out) + } + } +} diff --git a/common/serialize/bitset/bitset64.go b/common/serialize/bitset/bitset64.go new file mode 100644 index 000000000..e3c638470 --- /dev/null +++ b/common/serialize/bitset/bitset64.go @@ -0,0 +1,46 @@ +package bitset32 + +import ( + bitset64 "github.com/bits-and-blooms/bitset" +) + +type BitSet64 struct { + *bitset64.BitSet +} + +// TODO: TestFunc +func (b *BitSet64) MaxConsecutiveOne(start, end uint) uint { + return b.continueMaxCount(start, end, true) +} + +func (b *BitSet64) MaxConsecutiveZero(start, end uint) uint { + return b.continueMaxCount(start, end, false) +} + +func (b *BitSet64) continueMaxCount(start, end uint, flag bool) uint { + flag = !flag + if end > b.Len() { + end = b.Len() + } + if start >= b.Len() { + return 0 + } + if start > end { + return 0 + } + rt, sum := uint(0), uint(0) + for i := start; i < end; i++ { + if xor(flag, b.Test(i)) { + sum++ + continue + } + if sum > rt { + rt = sum + } + sum = 0 + } + if sum > rt { + rt = sum + } + return rt +} diff --git a/common/serialize/bitset/bitset_random_test.go b/common/serialize/bitset/bitset_random_test.go new file mode 100644 index 000000000..c7713d3f6 --- /dev/null +++ b/common/serialize/bitset/bitset_random_test.go @@ -0,0 +1,174 @@ +package bitset32 + +import ( + "math" + "math/rand" + "testing" + "time" + + "github.com/bits-and-blooms/bitset" +) + +var opc int +var opcT int + +var ft string = "%v|pos:%9X|opc:%9X|result:%v\n" +var rt string = "%v|opc:%9d|pass:%9d\n" + +var opNum = 100 +var bitTestNum = 10000 +var randNum = math.MaxInt32 / 2 + +func TestBitSet(t *testing.T) { + var b32 = New(1) + var b64 = bitset.New(1) + res := true + pos := uint(0) + rand.Seed(time.Now().Unix()) + for j := 0; j < 1; j++ { + // Test, Set, + for i := 0; i < opNum; i++ { + pos = uint(rand.Intn(randNum)) + b32 = b32.Set(uint(pos)) + b64 = b64.Set(uint(pos)) + res = b32.Test(pos) == b64.Test(pos) + opc++ + if res { + opcT++ + } else { + t.Log(ft, time.Now(), pos, opc, res) + } + } + // Clear + for i := 0; i < opNum; i++ { + pos = uint(rand.Intn(randNum)) + b32 = b32.Clear(uint(pos)) + b64 = b64.Clear(uint(pos)) + res = b32.Test(pos) == b64.Test(pos) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + // SetTo = Set + Clear + for i := 0; i < opNum; i++ { + pos = uint(rand.Intn(randNum)) + value := rand.Intn(randNum)%2 == 1 + b32 = b32.SetTo(uint(pos), value) + b64 = b64.SetTo(uint(pos), value) + res = b32.Test(pos) == b64.Test(pos) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + // Flip + for i := 0; i < opNum; i++ { + pos = uint(rand.Intn(randNum)) + b64 = b64.Flip(pos) + b32 = b32.Flip(pos) + res = isSameBitset(b32, b64) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + // Flip Range + for i := 0; i < opNum; i++ { + start, end := uint(rand.Intn(randNum)), uint(rand.Intn(randNum)) + if start > end { + start, end = end, start + } + b64 = b64.FlipRange(start, end) + b32 = b32.FlipRange(start, end) + res = isSameBitset(b32, b64) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + // InsertAt + for i := 0; i < opNum; i++ { + pos = uint(rand.Intn(randNum)) + b64 = b64.InsertAt(pos) + b32 = b32.InsertAt(pos) + res = isSameBitset(b32, b64) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + // DeleteAt + for i := 0; i < opNum; i++ { + pos = uint(rand.Intn(randNum)) + if b64.Len() < pos || b32.Len() < pos { + continue + } + b64 = b64.DeleteAt(pos) + b32 = b32.DeleteAt(pos) + res = isSameBitset(b32, b64) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + // Compact, Shrink + for i := 0; i < opNum; i++ { + b32 = b32.Compact() + b64 = b64.Compact() + res = isSameBitset(b32, b64) + opc++ + if res { + opcT++ + } else { + t.Logf(ft, time.Now().Unix(), pos, opc, res) + } + } + } + // Compact + b32 = b32.Compact() + b64 = b64.Compact() + res = isSameBitset(b32, b64) + t.Log("Compact:", res) + bs64 := &BitSet64{b64} + t.Log("Max Count:", b32.MaxConsecutiveOne(0, b32.Len()), bs64.MaxConsecutiveOne(0, b64.Len())) + t.Log("String:", b32.String() == b64.String()) + t.Logf(rt, time.Now().Unix(), opc, opcT) +} + +func isSameBitset(b32 *BitSet32, b64 *bitset.BitSet) bool { + if b32.Len() != b64.Len() { + return false + } + for i := 0; i < bitTestNum; i++ { + pos := uint(rand.Intn(randNum)) + if b32.Test(pos) != b64.Test(pos) { + return false + } + } + return true +} + +/* +Running tool: C:\support\go\bin\go.exe test -timeout 30s -run ^TestBitSet$ bitset -v + +=== RUN TestBitSet + d:\workspace\DataStruct\go\bitset\bitset_test.go:139: Max Count: 23067608 23067608 + d:\workspace\DataStruct\go\bitset\bitset_test.go:140: true + d:\workspace\DataStruct\go\bitset\bitset_test.go:141: 1678626066|opc: 800|pass: 800 +--- PASS: TestBitSet (23.61s) +PASS +ok bitset 24.243s +*/ diff --git a/common/serialize/bitset/go.mod b/common/serialize/bitset/go.mod new file mode 100644 index 000000000..2a2afd41f --- /dev/null +++ b/common/serialize/bitset/go.mod @@ -0,0 +1,5 @@ +module github.com/pointernil/bitset32 + +go 1.19 + +require github.com/bits-and-blooms/bitset v1.5.0 // direct diff --git a/common/serialize/bitset/go.sum b/common/serialize/bitset/go.sum new file mode 100644 index 000000000..686b4ccd3 --- /dev/null +++ b/common/serialize/bitset/go.sum @@ -0,0 +1,2 @@ +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= diff --git a/common/serialize/bitset/popcnt_19.go b/common/serialize/bitset/popcnt_19.go new file mode 100644 index 000000000..64c462b5f --- /dev/null +++ b/common/serialize/bitset/popcnt_19.go @@ -0,0 +1,43 @@ +package bitset32 + +import "math/bits" + +func popcntSlice(s []uint32) uint64 { + var cnt int + for _, x := range s { + cnt += bits.OnesCount32(x) + } + return uint64(cnt) +} + +func popcntMaskSlice(s, m []uint32) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount32(s[i] &^ m[i]) + } + return uint64(cnt) +} + +func popcntAndSlice(s, m []uint32) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount32(s[i] & m[i]) + } + return uint64(cnt) +} + +func popcntOrSlice(s, m []uint32) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount32(s[i] | m[i]) + } + return uint64(cnt) +} + +func popcntXorSlice(s, m []uint32) uint64 { + var cnt int + for i := range s { + cnt += bits.OnesCount32(s[i] ^ m[i]) + } + return uint64(cnt) +} diff --git a/common/serialize/bitset/trailingZeroes32.go b/common/serialize/bitset/trailingZeroes32.go new file mode 100644 index 000000000..622f1dab6 --- /dev/null +++ b/common/serialize/bitset/trailingZeroes32.go @@ -0,0 +1,10 @@ +//go:build go1.9 +// +build go1.9 + +package bitset32 + +import "math/bits" + +func trailingZeroes32(v uint32) uint { + return uint(bits.TrailingZeros32(v)) +} diff --git a/common/serialize/log/.gitignore b/common/serialize/log/.gitignore new file mode 100644 index 000000000..bf9b3b93e --- /dev/null +++ b/common/serialize/log/.gitignore @@ -0,0 +1,19 @@ +# Binaries for programs and plugins +*.exe +*.dll +*.so +*.dylib + +# Development binary, built with makefile +*.dev + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# IDE +.idea/* +.vscode/* +.history diff --git a/common/serialize/log/.travis.yml b/common/serialize/log/.travis.yml new file mode 100644 index 000000000..db15be69c --- /dev/null +++ b/common/serialize/log/.travis.yml @@ -0,0 +1,9 @@ +language: go +go: + - 1.6 + - 1.7 + - tip + +script: +- go vet $(go list ./...|grep -v "/vendor/") +- go test -v -race ./... diff --git a/common/serialize/log/LICENSE b/common/serialize/log/LICENSE new file mode 100644 index 000000000..8f71f43fe --- /dev/null +++ b/common/serialize/log/LICENSE @@ -0,0 +1,202 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + diff --git a/common/serialize/log/README.md b/common/serialize/log/README.md new file mode 100644 index 000000000..5070b808f --- /dev/null +++ b/common/serialize/log/README.md @@ -0,0 +1,103 @@ +# Termtables + +[![Build Status](https://travis-ci.org/scylladb/termtables.svg?branch=master)](https://travis-ci.org/scylladb/termtables) + +A [Go](http://golang.org) port of the Ruby library [terminal-tables](https://github.com/visionmedia/terminal-table) for +fast and simple ASCII table generation. + +## Installation + +```bash +go get github.com/scylladb/termtables +``` + +## Go Style Documentation + +[http://godoc.org/github.com/scylladb/termtables](http://godoc.org/github.com/scylladb/termtables) + +## APC Command Line usage + +`--markdown` — output a markdown table, e.g. `apc app list --markdown` + +`--html` — output an html table, e.g. `apc app list --html` + +`--ascii` — output an ascii table, e.g. `apc app list --ascii` + +## Basic Usage + +```go +package main + +import ( + "fmt" + "github.com/apcera/termtables" +) + +func main() { + table := termtables.CreateTable() + + table.AddHeaders("Name", "Age") + table.AddRow("John", "30") + table.AddRow("Sam", 18) + table.AddRow("Julie", 20.14) + + fmt.Println(table.Render()) +} +``` + +Result: + +``` ++-------+-------+ +| Name | Age | ++-------+-------+ +| John | 30 | +| Sam | 18 | +| Julie | 20.14 | ++-------+-------+ +``` + +## Advanced Usage + +The package function-call `EnableUTF8()` will cause any tables created after +that point to use Unicode box-drawing characters for the table lines. + +Calling `EnableUTF8PerLocale()` uses the C library's locale functionality to +determine if the current locale environment variables say that the current +character map is UTF-8. If, and only if, so, then `EnableUTF8()` will be +called. + +Calling `SetModeHTML(true)` will cause any tables created after that point +to be emitted in HTML, while `SetModeMarkdown(true)` will trigger Markdown. +Neither should result in changes to later API to get the different results; +the primary intended use-case is extracting the same table, but for +documentation. + +The table method `.AddSeparator()` inserts a rule line in the output. This +only applies in normal terminal output mode. + +The table method `.AddTitle()` adds a title to the table; in terminal output, +this is an initial row; in HTML, it's a caption. In Markdown, it's a line of +text before the table, prefixed by `Table: `. + +The table method `.SetAlign()` takes an alignment and a column number +(indexing starts at 1) and changes all _current_ cells in that column to have +the given alignment. It does not change the alignment of cells added to the +table after this call. Alignment is only stored on a per-cell basis. + +## Known Issues + +Normal output: + +* `.SetAlign()` does not affect headers. + +Markdown output mode: + +* When emitting Markdown, the column markers are not re-flowed if a vertical + bar is an element of a cell, causing an escape to take place; since Markdown + is often converted to HTML, this only affects text viewing. +* A title in Markdown is not escaped against all possible forms of Markdown + markup (to avoid adding a dependency upon a Markdown library, as supported + syntax can vary). +* Markdown requires headers, so a dummy header will be inserted if needed. +* Table alignment is not reflected in Markdown output. diff --git a/common/serialize/log/cell.go b/common/serialize/log/cell.go new file mode 100644 index 000000000..e1717504b --- /dev/null +++ b/common/serialize/log/cell.go @@ -0,0 +1,168 @@ +// Copyright 2012 Apcera Inc. All rights reserved. + +package termtables + +import ( + "fmt" + "math" + "regexp" + "strconv" + "strings" + "unicode/utf8" + + runewidth "github.com/mattn/go-runewidth" +) + +var ( + // Must match SGR escape sequence, which is "CSI Pm m", where the Control + // Sequence Introducer (CSI) is "ESC ["; where Pm is "A multiple numeric + // parameter composed of any number of single numeric parameters, separated + // by ; character(s). Individual values for the parameters are listed with + // Ps" and where Ps is A single (usually optional) numeric parameter, + // composed of one of [sic] more digits." + // + // In practice, the end sequence is usually given as \e[0m but reading that + // definition, it's clear that the 0 is optional and some testing confirms + // that it is certainly optional with MacOS Terminal 2.3, so we need to + // support the string \e[m as a terminator too. + colorFilter = regexp.MustCompile(`\033\[(?:\d+(?:;\d+)*)?m`) +) + +// A Cell denotes one cell of a table; it spans one row and a variable number +// of columns. A given Cell can only be used at one place in a table; the act +// of adding the Cell to the table mutates it with position information, so +// do not create one "const" Cell to add it multiple times. +type Cell struct { + column int + formattedValue string + alignment *TableAlignment + colSpan int +} + +// CreateCell returns a Cell where the content is the supplied value, with the +// optional supplied style (which may be given as nil). The style can include +// a non-zero ColSpan to cause the cell to become column-spanning. Changing +// the style afterwards will not adjust the column-spanning state of the cell +// itself. +func CreateCell(v interface{}, style *CellStyle) *Cell { + return createCell(0, v, style) +} + +func createCell(column int, v interface{}, style *CellStyle) *Cell { + cell := &Cell{column: column, formattedValue: renderValue(v), colSpan: 1} + if style != nil { + cell.alignment = &style.Alignment + if style.ColSpan != 0 { + cell.colSpan = style.ColSpan + } + } + return cell +} + +// Width returns the width of the content of the cell, measured in runes as best +// as possible considering sophisticated Unicode. +func (c *Cell) Width() int { + return runewidth.StringWidth(filterColorCodes(c.formattedValue)) +} + +// Filter out terminal bold/color sequences in a string. +// This supports only basic bold/color escape sequences. +func filterColorCodes(s string) string { + return colorFilter.ReplaceAllString(s, "") +} + +// Render returns a string representing the content of the cell, together with +// padding (to the widths specified) and handling any alignment. +func (c *Cell) Render(style *renderStyle) (buffer string) { + // if no alignment is set, import the table's default + if c.alignment == nil { + c.alignment = &style.Alignment + } + + // left padding + buffer += strings.Repeat(" ", style.PaddingLeft) + + // append the main value and handle alignment + buffer += c.alignCell(style) + + // right padding + buffer += strings.Repeat(" ", style.PaddingRight) + + // this handles escaping for, eg, Markdown, where we don't care about the + // alignment quite as much + if style.replaceContent != nil { + buffer = style.replaceContent(buffer) + } + + return buffer +} + +func (c *Cell) alignCell(style *renderStyle) string { + buffer := "" + width := style.CellWidth(c.column) + + if c.colSpan > 1 { + for i := 1; i < c.colSpan; i++ { + w := style.CellWidth(c.column + i) + if w == 0 { + break + } + width += style.PaddingLeft + w + style.PaddingRight + utf8.RuneCountInString(style.BorderY) + } + } + + switch *c.alignment { + + default: + buffer += c.formattedValue + if l := width - c.Width(); l > 0 { + buffer += strings.Repeat(" ", l) + } + + case AlignLeft: + buffer += c.formattedValue + if l := width - c.Width(); l > 0 { + buffer += strings.Repeat(" ", l) + } + + case AlignRight: + if l := width - c.Width(); l > 0 { + buffer += strings.Repeat(" ", l) + } + buffer += c.formattedValue + + case AlignCenter: + left, right := 0, 0 + if l := width - c.Width(); l > 0 { + lf := float64(l) + left = int(math.Floor(lf / 2)) + right = int(math.Ceil(lf / 2)) + } + buffer += strings.Repeat(" ", left) + buffer += c.formattedValue + buffer += strings.Repeat(" ", right) + } + + return buffer +} + +// Format the raw value as a string depending on the type +func renderValue(v interface{}) string { + switch vv := v.(type) { + case string: + return vv + case bool: + return strconv.FormatBool(vv) + case int: + return strconv.Itoa(vv) + case int64: + return strconv.FormatInt(vv, 10) + case uint64: + return strconv.FormatUint(vv, 10) + case float64: + return strconv.FormatFloat(vv, 'f', 2, 64) + case fmt.Stringer: + return vv.String() + } + return fmt.Sprintf("%v", v) +} diff --git a/common/serialize/log/cell_test.go b/common/serialize/log/cell_test.go new file mode 100644 index 000000000..e091679b4 --- /dev/null +++ b/common/serialize/log/cell_test.go @@ -0,0 +1,113 @@ +// Copyright 2012-2015 Apcera Inc. All rights reserved. + +package termtables + +import ( + "testing" +) + +func TestCellRenderString(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{}, cellWidths: map[int]int{}} + cell := createCell(0, "foobar", nil) + + output := cell.Render(style) + if output != "foobar" { + t.Fatal("Unexpected output:", output) + } +} + +func TestCellRenderBool(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{}, cellWidths: map[int]int{}} + cell := createCell(0, true, nil) + + output := cell.Render(style) + if output != "true" { + t.Fatal("Unexpected output:", output) + } +} + +func TestCellRenderInteger(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{}, cellWidths: map[int]int{}} + cell := createCell(0, 12345, nil) + + output := cell.Render(style) + if output != "12345" { + t.Fatal("Unexpected output:", output) + } +} + +func TestCellRenderFloat(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{}, cellWidths: map[int]int{}} + cell := createCell(0, 12.345, nil) + + output := cell.Render(style) + if output != "12.35" { + t.Fatal("Unexpected output:", output) + } +} + +func TestCellRenderPadding(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{PaddingLeft: 3, PaddingRight: 4}, cellWidths: map[int]int{}} + + cell := createCell(0, "foobar", nil) + + output := cell.Render(style) + if output != " foobar " { + t.Fatal("Unexpected output:", output) + } +} + +type foo struct { + v string +} + +func (f *foo) String() string { + return f.v +} + +func TestCellRenderStringerStruct(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{}, cellWidths: map[int]int{}} + cell := createCell(0, &foo{v: "bar"}, nil) + + output := cell.Render(style) + if output != "bar" { + t.Fatal("Unexpected output:", output) + } +} + +type fooString string + +func TestCellRenderGeneric(t *testing.T) { + style := &renderStyle{TableStyle: TableStyle{}, cellWidths: map[int]int{}} + cell := createCell(0, fooString("baz"), nil) + + output := cell.Render(style) + if output != "baz" { + t.Fatal("Unexpected output:", output) + } +} + +func TestFilterColorCodes(t *testing.T) { + tests := []struct { + in string + out string + }{ + {"abc", "abc"}, + {"", ""}, + {"\033[31m\033[0m", ""}, + {"a\033[31mb\033[0mc", "abc"}, + {"\033[31mabc\033[0m", "abc"}, + {"\033[31mfoo\033[0mbar", "foobar"}, + {"\033[31mfoo\033[mbar", "foobar"}, + {"\033[31mfoo\033[0;0mbar", "foobar"}, + {"\033[31;4mfoo\033[0mbar", "foobar"}, + {"\033[31;4;43mfoo\033[0mbar", "foobar"}, + } + for _, test := range tests { + got := filterColorCodes(test.in) + if got != test.out { + t.Errorf("Invalid color-code filter result; expected %q but got %q from input %q", + test.out, got, test.in) + } + } +} diff --git a/common/serialize/log/go.mod b/common/serialize/log/go.mod new file mode 100644 index 000000000..b49885e10 --- /dev/null +++ b/common/serialize/log/go.mod @@ -0,0 +1,8 @@ +module github.com/apcera/termtables + +go 1.20 + +require ( + github.com/mattn/go-runewidth v0.0.3-0.20170201023540-14207d285c6c + github.com/scylladb/termtables v1.0.0 +) diff --git a/common/serialize/log/go.sum b/common/serialize/log/go.sum new file mode 100644 index 000000000..26e6deb5e --- /dev/null +++ b/common/serialize/log/go.sum @@ -0,0 +1,4 @@ +github.com/mattn/go-runewidth v0.0.3-0.20170201023540-14207d285c6c h1:jQ6tSGsM/2TGhmbzHl9wXDtm2YjZDAfMsHyxaBDwywA= +github.com/mattn/go-runewidth v0.0.3-0.20170201023540-14207d285c6c/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/scylladb/termtables v1.0.0 h1:uUnesUY4V1VPCotpOQLb1LjTXVvzwy7Ramx8K8+w+8U= +github.com/scylladb/termtables v1.0.0/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= diff --git a/common/serialize/log/html.go b/common/serialize/log/html.go new file mode 100644 index 000000000..027662e4b --- /dev/null +++ b/common/serialize/log/html.go @@ -0,0 +1,107 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +package termtables + +import ( + "bytes" + "fmt" + "html" + "strings" +) + +type titleStyle int + +const ( + TitleAsCaption titleStyle = iota + TitleAsThSpan +) + +// htmlStyleRules defines attributes which we can use, and might be set on a +// table by accessors, to influence the type of HTML which is output. +type htmlStyleRules struct { + title titleStyle +} + +// HTML returns an HTML representations of the contents of one row of a table. +func (r *Row) HTML(tag string, style *renderStyle) string { + attrs := make([]string, len(r.cells)) + elems := make([]string, len(r.cells)) + for i := range r.cells { + if r.cells[i].alignment != nil { + switch *r.cells[i].alignment { + case AlignLeft: + attrs[i] = " align='left'" + case AlignCenter: + attrs[i] = " align='center'" + case AlignRight: + attrs[i] = " align='right'" + } + } + elems[i] = html.EscapeString(strings.TrimSpace(r.cells[i].Render(style))) + } + // WAG as to max capacity, plus a bit + buf := bytes.NewBuffer(make([]byte, 0, 8192)) + buf.WriteString("") + for i := range elems { + fmt.Fprintf(buf, "<%s%s>%s", tag, attrs[i], elems[i], tag) + } + buf.WriteString("\n") + return buf.String() +} + +func generateHtmlTitleRow(title interface{}, t *Table, style *renderStyle) string { + elContent := html.EscapeString( + strings.TrimSpace(CreateCell(t.title, &CellStyle{}).Render(style)), + ) + + switch style.htmlRules.title { + case TitleAsCaption: + return "" + elContent + "\n" + case TitleAsThSpan: + return fmt.Sprintf("%s\n", + style.columns, elContent) + default: + return "" + } +} + +// RenderHTML returns a string representation of a the table, suitable for +// inclusion as HTML elsewhere. Primary use-case controlling layout style +// is for inclusion into Markdown documents, documenting normal table use. +// Thus we leave the padding in place to have columns align when viewed as +// plain text and rely upon HTML ignoring extra whitespace. +func (t *Table) RenderHTML() (buffer string) { + // elements is already populated with row data + + // generate the runtime style + style := createRenderStyle(t) + style.PaddingLeft = 0 + style.PaddingRight = 0 + + // TODO: control CSS styles to suppress border based upon t.Style.SkipBorder + rowsText := make([]string, 0, len(t.elements)+6) + + if t.title != nil || t.headers != nil { + rowsText = append(rowsText, "\n") + if t.title != nil { + rowsText = append(rowsText, generateHtmlTitleRow(t.title, t, style)) + } + if t.headers != nil { + rowsText = append(rowsText, CreateRow(t.headers).HTML("th", style)) + } + rowsText = append(rowsText, "\n") + } + + rowsText = append(rowsText, "\n") + // loop over the elements and render them + for i := range t.elements { + if row, ok := t.elements[i].(*Row); ok { + rowsText = append(rowsText, row.HTML("td", style)) + } else { + rowsText = append(rowsText, fmt.Sprintf("\n", i)) + } + } + rowsText = append(rowsText, "\n") + + return "\n" + strings.Join(rowsText, "") + "
\n" +} diff --git a/common/serialize/log/html_test.go b/common/serialize/log/html_test.go new file mode 100644 index 000000000..b9f70baaf --- /dev/null +++ b/common/serialize/log/html_test.go @@ -0,0 +1,222 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +package termtables + +import ( + "testing" +) + +func TestCreateTableHTML(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
NameValue
heyyou
ken1234
derek3.14
derek too3.15
\n" + + table := CreateTable() + table.SetModeHTML() + + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableWithHeaderHTML(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
Example
NameValue
heyyou
ken1234
derek3.14
derek too3.15
\n" + + table := CreateTable() + table.SetModeHTML() + + table.AddTitle("Example") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableTitleWidthAdjustsHTML(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
Example My Foo Bar'd Test
NameValue
heyyou
ken1234
derek3.14
derek too3.15
\n" + + table := CreateTable() + table.SetModeHTML() + + table.AddTitle("Example My Foo Bar'd Test") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableWithNoHeadersHTML(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
heyyou
ken1234
derek3.14
derek too3.15
\n" + + table := CreateTable() + table.SetModeHTML() + + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableUnicodeWidthsHTML(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
NameCost
Currency¤10
US Dollar$30
Euro€27
Thai฿70
\n" + + table := CreateTable() + table.SetModeHTML() + table.AddHeaders("Name", "Cost") + table.AddRow("Currency", "¤10") + table.AddRow("US Dollar", "$30") + table.AddRow("Euro", "€27") + table.AddRow("Thai", "฿70") + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableWithAlignment(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
FooBar
humptydumpty
r<- on right
\n" + + table := CreateTable() + table.SetModeHTML() + table.AddHeaders("Foo", "Bar") + table.AddRow("humpty", "dumpty") + table.AddRow(CreateCell("r", &CellStyle{Alignment: AlignRight}), "<- on right") + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableAfterSetAlign(t *testing.T) { + expected := "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
AlphabeticalNum
alfa1
bravo2
charlie3
\n" + + table := CreateTable() + table.SetModeHTML() + table.AddHeaders("Alphabetical", "Num") + table.AddRow("alfa", 1) + table.AddRow("bravo", 2) + table.AddRow("charlie", 3) + table.SetAlign(AlignRight, 1) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableWithAltTitleStyle(t *testing.T) { + expected := "" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "\n" + + "
Metasyntactic
FooBarBaz
abc
αβγ
\n" + + table := CreateTable() + table.SetModeHTML() + table.SetHTMLStyleTitle(TitleAsThSpan) + table.AddTitle("Metasyntactic") + table.AddHeaders("Foo", "Bar", "Baz") + table.AddRow("a", "b", "c") + table.AddRow("α", "β", "γ") + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} diff --git a/common/serialize/log/log.go b/common/serialize/log/log.go deleted file mode 100644 index 94b591687..000000000 --- a/common/serialize/log/log.go +++ /dev/null @@ -1,5 +0,0 @@ -package log - -func Init() { - -} diff --git a/common/serialize/log/log_test.go b/common/serialize/log/log_test.go deleted file mode 100644 index f0584eae5..000000000 --- a/common/serialize/log/log_test.go +++ /dev/null @@ -1,19 +0,0 @@ -package log - -import ( - "fmt" - "testing" - - "github.com/apcera/termtables" -) - -func TestInit(t *testing.T) { - table := termtables.CreateTable() - - table.AddHeaders("Name", "Age") - table.AddRow("John", "30") - table.AddRow("Sam", 18) - table.AddRow("Julie", 20.14) - - fmt.Println(table.Render()) -} diff --git a/common/serialize/log/row.go b/common/serialize/log/row.go new file mode 100644 index 000000000..fb53c01a9 --- /dev/null +++ b/common/serialize/log/row.go @@ -0,0 +1,47 @@ +// Copyright 2012 Apcera Inc. All rights reserved. + +package termtables + +import "strings" + +// A Row represents one row of a Table, consisting of some number of Cell +// items. +type Row struct { + cells []*Cell +} + +// CreateRow returns a Row where the cells are created as needed to hold each +// item given; each item can be a Cell or content to go into a Cell created +// to hold it. +func CreateRow(items []interface{}) *Row { + row := &Row{cells: []*Cell{}} + for _, item := range items { + row.AddCell(item) + } + return row +} + +// AddCell adds one item to a row as a new cell, where the item is either a +// Cell or content to be put into a cell. +func (r *Row) AddCell(item interface{}) { + if c, ok := item.(*Cell); ok { + c.column = len(r.cells) + r.cells = append(r.cells, c) + } else { + r.cells = append(r.cells, createCell(len(r.cells), item, nil)) + } +} + +// Render returns a string representing the content of one row of a table, where +// the Row contains Cells (not Separators) and the representation includes any +// vertical borders needed. +func (r *Row) Render(style *renderStyle) string { + // pre-render and shove into an array... helps with cleanly adding borders + renderedCells := []string{} + for _, c := range r.cells { + renderedCells = append(renderedCells, c.Render(style)) + } + + // format final output + return style.BorderY + strings.Join(renderedCells, style.BorderY) + style.BorderY +} diff --git a/common/serialize/log/row_test.go b/common/serialize/log/row_test.go new file mode 100644 index 000000000..f8f0bd289 --- /dev/null +++ b/common/serialize/log/row_test.go @@ -0,0 +1,29 @@ +// Copyright 2012-2015 Apcera Inc. All rights reserved. + +package termtables + +import ( + "testing" +) + +func TestBasicRowRender(t *testing.T) { + row := CreateRow([]interface{}{"foo", "bar"}) + style := &renderStyle{TableStyle: TableStyle{BorderX: "-", BorderY: "|", BorderI: "+", + PaddingLeft: 1, PaddingRight: 1}, cellWidths: map[int]int{0: 3, 1: 3}} + + output := row.Render(style) + if output != "| foo | bar |" { + t.Fatal("Unexpected output:", output) + } +} + +func TestRowRenderWidthBasedPadding(t *testing.T) { + row := CreateRow([]interface{}{"foo", "bar"}) + style := &renderStyle{TableStyle: TableStyle{BorderX: "-", BorderY: "|", BorderI: "+", + PaddingLeft: 1, PaddingRight: 1}, cellWidths: map[int]int{0: 3, 1: 5}} + + output := row.Render(style) + if output != "| foo | bar |" { + t.Fatal("Unexpected output:", output) + } +} diff --git a/common/serialize/log/separator.go b/common/serialize/log/separator.go new file mode 100644 index 000000000..686ef12e9 --- /dev/null +++ b/common/serialize/log/separator.go @@ -0,0 +1,60 @@ +// Copyright 2012 Apcera Inc. All rights reserved. + +package termtables + +import "strings" + +type lineType int + +// These lines are for horizontal rules; these indicate desired styling, +// but simplistic (pure ASCII) markup characters may end up leaving the +// variant lines indistinguishable from LINE_INNER. +const ( + // LINE_INNER *must* be the default; where there are vertical lines drawn + // across an inner line, the character at that position should indicate + // that the vertical line goes both up and down from this horizontal line. + LINE_INNER lineType = iota + + // LINE_TOP has only descenders + LINE_TOP + + // LINE_SUBTOP has only descenders in the middle, but goes both up and + // down at the far left & right edges. + LINE_SUBTOP + + // LINE_BOTTOM has only ascenders. + LINE_BOTTOM +) + +// A Separator is a horizontal rule line, with associated information which +// indicates where in a table it is, sufficient for simple cases to let +// clean tables be drawn. If a row-spanning cell is created, then this will +// be insufficient: we can get away with hand-waving of "well, it's showing +// where the border would be" but a more capable handling will require +// structure reworking. Patches welcome. +type Separator struct { + where lineType +} + +// Render returns the string representation of a horizontal rule line in the +// table. +func (s *Separator) Render(style *renderStyle) string { + // loop over getting dashes + parts := []string{} + for i := 0; i < style.columns; i++ { + w := style.PaddingLeft + style.CellWidth(i) + style.PaddingRight + parts = append(parts, strings.Repeat(style.BorderX, w)) + } + + switch s.where { + case LINE_TOP: + return style.BorderTopLeft + strings.Join(parts, style.BorderTop) + style.BorderTopRight + case LINE_SUBTOP: + return style.BorderLeft + strings.Join(parts, style.BorderTop) + style.BorderRight + case LINE_BOTTOM: + return style.BorderBottomLeft + strings.Join(parts, style.BorderBottom) + style.BorderBottomRight + case LINE_INNER: + return style.BorderLeft + strings.Join(parts, style.BorderI) + style.BorderRight + } + panic("not reached") +} diff --git a/common/serialize/log/straight_separator.go b/common/serialize/log/straight_separator.go new file mode 100644 index 000000000..9fd7e3b41 --- /dev/null +++ b/common/serialize/log/straight_separator.go @@ -0,0 +1,36 @@ +// Copyright 2012 Apcera Inc. All rights reserved. + +package termtables + +import ( + "strings" + "unicode/utf8" +) + +// A StraightSeparator is a horizontal line with associated information about +// what sort of position it takes in the table, so as to control which shapes +// will be used where vertical lines are expected to touch this horizontal +// line. +type StraightSeparator struct { + where lineType +} + +// Render returns a string representing this separator, with all border +// crossings appropriately chosen. +func (s *StraightSeparator) Render(style *renderStyle) string { + // loop over getting dashes + width := 0 + for i := 0; i < style.columns; i++ { + width += style.PaddingLeft + style.CellWidth(i) + style.PaddingRight + utf8.RuneCountInString(style.BorderI) + } + + switch s.where { + case LINE_TOP: + return style.BorderTopLeft + strings.Repeat(style.BorderX, width-1) + style.BorderTopRight + case LINE_INNER, LINE_SUBTOP: + return style.BorderLeft + strings.Repeat(style.BorderX, width-1) + style.BorderRight + case LINE_BOTTOM: + return style.BorderBottomLeft + strings.Repeat(style.BorderX, width-1) + style.BorderBottomRight + } + panic("not reached") +} diff --git a/common/serialize/log/style.go b/common/serialize/log/style.go new file mode 100644 index 000000000..9ce1bab1d --- /dev/null +++ b/common/serialize/log/style.go @@ -0,0 +1,214 @@ +// Copyright 2012-2013 Apcera Inc. All rights reserved. + +package termtables + +import ( + "fmt" + "strings" + "unicode/utf8" +) + +type TableAlignment int + +// These constants control the alignment which should be used when rendering +// the content of a cell. +const ( + AlignLeft = TableAlignment(1) + AlignCenter = TableAlignment(2) + AlignRight = TableAlignment(3) +) + +// TableStyle controls styling information for a Table as a whole. +// +// For the Border rules, only X, Y and I are needed, and all have defaults. +// The others will all default to the same as BorderI. +type TableStyle struct { + SkipBorder bool + BorderX string + BorderY string + BorderI string + BorderTop string + BorderBottom string + BorderRight string + BorderLeft string + BorderTopLeft string + BorderTopRight string + BorderBottomLeft string + BorderBottomRight string + PaddingLeft int + PaddingRight int + Width int + Alignment TableAlignment + htmlRules htmlStyleRules +} + +// A CellStyle controls all style applicable to one Cell. +type CellStyle struct { + // Alignment indicates the alignment to be used in rendering the content + Alignment TableAlignment + + // ColSpan indicates how many columns this Cell is expected to consume. + ColSpan int +} + +// DefaultStyle is a TableStyle which can be used to get some simple +// default styling for a table, using ASCII characters for drawing borders. +var DefaultStyle = &TableStyle{ + SkipBorder: false, + BorderX: "-", BorderY: "|", BorderI: "+", + PaddingLeft: 1, PaddingRight: 1, + Width: 80, + Alignment: AlignLeft, + + // FIXME: the use of a Width here may interact poorly with a changing + // MaxColumns value; we don't set MaxColumns here because the evaluation + // order of a var and an init value adds undesired subtlety. +} + +type renderStyle struct { + cellWidths map[int]int + columns int + + // used for markdown rendering + replaceContent func(string) string + + TableStyle +} + +// setUtfBoxStyle changes the border characters to be suitable for use when +// the output stream can render UTF-8 characters. +func (s *TableStyle) setUtfBoxStyle() { + s.BorderX = "─" + s.BorderY = "│" + s.BorderI = "┼" + s.BorderTop = "┬" + s.BorderBottom = "┴" + s.BorderLeft = "├" + s.BorderRight = "┤" + s.BorderTopLeft = "╭" + s.BorderTopRight = "╮" + s.BorderBottomLeft = "╰" + s.BorderBottomRight = "╯" +} + +// setAsciiBoxStyle changes the border characters back to their defaults +func (s *TableStyle) setAsciiBoxStyle() { + s.BorderX = "-" + s.BorderY = "|" + s.BorderI = "+" + s.BorderTop, s.BorderBottom, s.BorderLeft, s.BorderRight = "", "", "", "" + s.BorderTopLeft, s.BorderTopRight, s.BorderBottomLeft, s.BorderBottomRight = "", "", "", "" + s.fillStyleRules() +} + +// fillStyleRules populates members of the TableStyle box-drawing specification +// with BorderI as the default. +func (s *TableStyle) fillStyleRules() { + if s.BorderTop == "" { + s.BorderTop = s.BorderI + } + if s.BorderBottom == "" { + s.BorderBottom = s.BorderI + } + if s.BorderLeft == "" { + s.BorderLeft = s.BorderI + } + if s.BorderRight == "" { + s.BorderRight = s.BorderI + } + if s.BorderTopLeft == "" { + s.BorderTopLeft = s.BorderI + } + if s.BorderTopRight == "" { + s.BorderTopRight = s.BorderI + } + if s.BorderBottomLeft == "" { + s.BorderBottomLeft = s.BorderI + } + if s.BorderBottomRight == "" { + s.BorderBottomRight = s.BorderI + } +} + +func createRenderStyle(table *Table) *renderStyle { + style := &renderStyle{TableStyle: *table.Style, cellWidths: map[int]int{}} + style.TableStyle.fillStyleRules() + + if table.outputMode == outputMarkdown { + style.buildReplaceContent(table.Style.BorderY) + } + + // FIXME: handle actually defined width condition + + // loop over the rows and cells to calculate widths + for _, element := range table.elements { + // skip separators + if _, ok := element.(*Separator); ok { + continue + } + + // iterate over cells + if row, ok := element.(*Row); ok { + for i, cell := range row.cells { + // FIXME: need to support sizing with colspan handling + if cell.colSpan > 1 { + continue + } + if style.cellWidths[i] < cell.Width() { + style.cellWidths[i] = cell.Width() + } + } + } + } + style.columns = len(style.cellWidths) + + // calculate actual width + width := utf8.RuneCountInString(style.BorderLeft) // start at '1' for left border + internalBorderWidth := utf8.RuneCountInString(style.BorderI) + + lastIndex := 0 + for i, v := range style.cellWidths { + width += v + style.PaddingLeft + style.PaddingRight + internalBorderWidth + if i > lastIndex { + lastIndex = i + } + } + if internalBorderWidth != utf8.RuneCountInString(style.BorderRight) { + width += utf8.RuneCountInString(style.BorderRight) - internalBorderWidth + } + + if table.titleCell != nil { + titleMinWidth := 0 + + table.titleCell.Width() + + utf8.RuneCountInString(style.BorderLeft) + + utf8.RuneCountInString(style.BorderRight) + + style.PaddingLeft + + style.PaddingRight + + if width < titleMinWidth { + // minWidth must be set to include padding of the title, as required + style.cellWidths[lastIndex] += (titleMinWidth - width) + width = titleMinWidth + } + } + + // right border is covered in loop + style.Width = width + + return style +} + +// CellWidth returns the width of the cell at the supplied index, where the +// width is the number of tty character-cells required to draw the glyphs. +func (s *renderStyle) CellWidth(i int) int { + return s.cellWidths[i] +} + +// buildReplaceContent creates a function closure, with minimal bound lexical +// state, which replaces content +func (s *renderStyle) buildReplaceContent(bad string) { + replacement := fmt.Sprintf("&#x%02x;", bad) + s.replaceContent = func(old string) string { + return strings.Replace(old, bad, replacement, -1) + } +} diff --git a/common/serialize/log/table.go b/common/serialize/log/table.go new file mode 100644 index 000000000..67d2e18db --- /dev/null +++ b/common/serialize/log/table.go @@ -0,0 +1,373 @@ +// Copyright 2012-2013 Apcera Inc. All rights reserved. + +package termtables + +import ( + "bytes" + "os" + "regexp" + "runtime" + "strings" + + "github.com/apcera/termtables/term" +) + +// MaxColumns represents the maximum number of columns that are available for +// display without wrapping around the right-hand side of the terminal window. +// At program initialization, the value will be automatically set according +// to available sources of information, including the $COLUMNS environment +// variable and, on Unix, tty information. +var MaxColumns = 80 + +// Element the interface that can draw a representation of the contents of a +// table cell. +type Element interface { + Render(*renderStyle) string +} + +type outputMode int + +const ( + outputTerminal outputMode = iota + outputMarkdown + outputHTML +) + +// Open question: should UTF-8 become an output mode? It does require more +// tracking when resetting, if the locale-enabling had been used. + +var outputsEnabled struct { + UTF8 bool + HTML bool + Markdown bool + titleStyle titleStyle +} + +var defaultOutputMode outputMode = outputTerminal + +// Table represents a terminal table. The Style can be directly accessed +// and manipulated; all other access is via methods. +type Table struct { + Style *TableStyle + + elements []Element + headers []interface{} + title interface{} + titleCell *Cell + outputMode outputMode +} + +// EnableUTF8 will unconditionally enable using UTF-8 box-drawing characters +// for any tables created after this call, as the default style. +func EnableUTF8() { + outputsEnabled.UTF8 = true +} + +// SetModeHTML will control whether or not new tables generated will be in HTML +// mode by default; HTML-or-not takes precedence over options which control how +// a terminal output will be rendered, such as whether or not to use UTF8. +// This affects any tables created after this call. +func SetModeHTML(onoff bool) { + outputsEnabled.HTML = onoff + chooseDefaultOutput() +} + +// SetModeMarkdown will control whether or not new tables generated will be +// in Markdown mode by default. HTML-mode takes precedence. +func SetModeMarkdown(onoff bool) { + outputsEnabled.Markdown = onoff + chooseDefaultOutput() +} + +var utfRe = regexp.MustCompile(`utf\-8|utf8|UTF\-8|UTF8`) + +// EnableUTF8PerLocale will use current locale character map information to +// determine if UTF-8 is expected and, if so, is equivalent to EnableUTF8. +func EnableUTF8PerLocale() { + locale := getLocale() + if utfRe.MatchString(locale) { + EnableUTF8() + } +} + +// getLocale returns the current locale name. +func getLocale() string { + if runtime.GOOS == "windows" { + // TODO: detect windows locale + return "US-ASCII" + } + return unixLocale() +} + +// unixLocale returns the locale by checking the $LC_ALL, $LC_CTYPE, and $LANG +// environment variables. If none of those are set, it returns "US-ASCII". +func unixLocale() string { + for _, env := range []string{"LC_ALL", "LC_CTYPE", "LANG"} { + if locale := os.Getenv(env); locale != "" { + return locale + } + } + return "US-ASCII" +} + +// SetHTMLStyleTitle lets an HTML title output mode be chosen. +func SetHTMLStyleTitle(want titleStyle) { + outputsEnabled.titleStyle = want +} + +// chooseDefaultOutput sets defaultOutputMode based on priority +// choosing amongst the options which are enabled. Pros: simpler +// encapsulation; cons: setting markdown doesn't disable HTML if +// HTML was previously enabled and was later disabled. +// This seems fairly reasonable. +func chooseDefaultOutput() { + if outputsEnabled.HTML { + defaultOutputMode = outputHTML + } else if outputsEnabled.Markdown { + defaultOutputMode = outputMarkdown + } else { + defaultOutputMode = outputTerminal + } +} + +func init() { + // Do not enable UTF-8 per locale by default, breaks tests. + sz, err := term.GetSize() + if err == nil && sz.Columns != 0 { + MaxColumns = sz.Columns + } +} + +// CreateTable creates an empty Table using defaults for style. +func CreateTable() *Table { + t := &Table{elements: []Element{}, Style: DefaultStyle} + if outputsEnabled.UTF8 { + t.Style.setUtfBoxStyle() + } + if outputsEnabled.titleStyle != titleStyle(0) { + t.Style.htmlRules.title = outputsEnabled.titleStyle + } + t.outputMode = defaultOutputMode + return t +} + +// AddSeparator adds a line to the table content, where the line +// consists of separator characters. +func (t *Table) AddSeparator() { + t.elements = append(t.elements, &Separator{}) +} + +// AddRow adds the supplied items as cells in one row of the table. +func (t *Table) AddRow(items ...interface{}) *Row { + row := CreateRow(items) + t.elements = append(t.elements, row) + return row +} + +// AddTitle supplies a table title, which if present will be rendered as +// one cell across the width of the table, as the first row. +func (t *Table) AddTitle(title interface{}) { + t.title = title +} + +// AddHeaders supplies column headers for the table. +func (t *Table) AddHeaders(headers ...interface{}) { + t.headers = append(t.headers, headers...) +} + +// SetAlign changes the alignment for elements in a column of the table; +// alignments are stored with each cell, so cells added after a call to +// SetAlign will not pick up the change. Columns are numbered from 1. +func (t *Table) SetAlign(align TableAlignment, columns ...int) { + for i := range t.elements { + row, ok := t.elements[i].(*Row) + if !ok { + continue + } + for _, column := range columns { + if column < 0 || column > len(row.cells) { + continue + } + row.cells[column-1].alignment = &align + } + } +} + +// UTF8Box sets the table style to use UTF-8 box-drawing characters, +// overriding all relevant style elements at the time of the call. +func (t *Table) UTF8Box() { + t.Style.setUtfBoxStyle() +} + +// SetModeHTML switches this table to be in HTML when rendered; the +// default depends upon whether the package function SetModeHTML() has been +// called, and with what value. This method forces the feature on for this +// table. Turning off involves choosing a different mode, per-table. +func (t *Table) SetModeHTML() { + t.outputMode = outputHTML +} + +// SetModeMarkdown switches this table to be in Markdown mode +func (t *Table) SetModeMarkdown() { + t.outputMode = outputMarkdown +} + +// SetModeTerminal switches this table to be in terminal mode. +func (t *Table) SetModeTerminal() { + t.outputMode = outputTerminal +} + +// SetHTMLStyleTitle lets an HTML output mode be chosen; we should rework this +// into a more generic and extensible API as we clean up termtables. +func (t *Table) SetHTMLStyleTitle(want titleStyle) { + t.Style.htmlRules.title = want +} + +// Render returns a string representation of a fully rendered table, drawn +// out for display, with embedded newlines. If this table is in HTML mode, +// then this is equivalent to RenderHTML(). +func (t *Table) Render() string { + // Elements is already populated with row data. + switch t.outputMode { + case outputTerminal: + return t.renderTerminal() + case outputMarkdown: + return t.renderMarkdown() + case outputHTML: + return t.RenderHTML() + default: + panic("unknown output mode set") + } +} + +// renderTerminal returns a string representation of a fully rendered table, +// drawn out for display, with embedded newlines. +func (t *Table) renderTerminal() string { + // Use a placeholder rather than adding titles/headers to the tables + // elements or else successive calls will compound them. + tt := t.clone() + + // Initial top line. + if !tt.Style.SkipBorder { + if tt.title != nil && tt.headers == nil { + tt.elements = append([]Element{&Separator{where: LINE_SUBTOP}}, tt.elements...) + } else if tt.title == nil && tt.headers == nil { + tt.elements = append([]Element{&Separator{where: LINE_TOP}}, tt.elements...) + } else { + tt.elements = append([]Element{&Separator{where: LINE_INNER}}, tt.elements...) + } + } + + // If we have headers, include them. + if tt.headers != nil { + ne := make([]Element, 2) + ne[1] = CreateRow(tt.headers) + if tt.title != nil { + ne[0] = &Separator{where: LINE_SUBTOP} + } else { + ne[0] = &Separator{where: LINE_TOP} + } + tt.elements = append(ne, tt.elements...) + } + + // If we have a title, write it. + if tt.title != nil { + // Match changes to this into renderMarkdown too. + tt.titleCell = CreateCell(tt.title, &CellStyle{Alignment: AlignCenter, ColSpan: 999}) + ne := []Element{ + &StraightSeparator{where: LINE_TOP}, + CreateRow([]interface{}{tt.titleCell}), + } + tt.elements = append(ne, tt.elements...) + } + + // Create a new table from the + // generate the runtime style. Must include all cells being printed. + style := createRenderStyle(tt) + + // Loop over the elements and render them. + b := bytes.NewBuffer(nil) + for _, e := range tt.elements { + b.WriteString(e.Render(style)) + b.WriteString("\n") + } + + // Add bottom line. + if !style.SkipBorder { + b.WriteString((&Separator{where: LINE_BOTTOM}).Render(style) + "\n") + } + + return b.String() +} + +// renderMarkdown returns a string representation of a table in Markdown +// markup format using GitHub Flavored Markdown's notation (since tables +// are not in the core Markdown spec). +func (t *Table) renderMarkdown() string { + // We need ASCII drawing characters; we need a line after the header; + // *do* need a header! Do not need to markdown-escape contents of + // tables as markdown is ignored in there. Do need to do _something_ + // with a '|' character shown as a member of a table. + + t.Style.setAsciiBoxStyle() + + firstLines := make([]Element, 0, 2) + + if t.headers == nil { + initial := createRenderStyle(t) + if initial.columns > 1 { + row := CreateRow([]interface{}{}) + for i := 0; i < initial.columns; i++ { + row.AddCell(CreateCell(i+1, &CellStyle{})) + } + } + } + + firstLines = append(firstLines, CreateRow(t.headers)) + // This is a dummy line, swapped out below. + firstLines = append(firstLines, firstLines[0]) + t.elements = append(firstLines, t.elements...) + // Generate the runtime style. + style := createRenderStyle(t) + // We know that the second line is a dummy, we can replace it. + mdRow := CreateRow([]interface{}{}) + for i := 0; i < style.columns; i++ { + mdRow.AddCell(CreateCell(strings.Repeat("-", style.cellWidths[i]), &CellStyle{})) + } + t.elements[1] = mdRow + + b := bytes.NewBuffer(nil) + // Comes after style is generated, which must come after all width-affecting + // changes are in. + if t.title != nil { + // Markdown doesn't support titles or column spanning; we _should_ + // escape the title, but doing that to handle all possible forms of + // markup would require a heavy dependency, so we punt. + b.WriteString("Table: ") + b.WriteString(strings.TrimSpace(CreateCell(t.title, &CellStyle{}).Render(style))) + b.WriteString("\n\n") + } + + // Loop over the elements and render them. + for _, e := range t.elements { + b.WriteString(e.Render(style)) + b.WriteString("\n") + } + + return b.String() +} + +// clone returns a copy of the table with the underlying slices being copied; +// the references to the Elements/cells are left as shallow copies. +func (t *Table) clone() *Table { + tt := &Table{outputMode: t.outputMode, Style: t.Style, title: t.title} + if t.headers != nil { + tt.headers = make([]interface{}, len(t.headers)) + copy(tt.headers, t.headers) + } + if t.elements != nil { + tt.elements = make([]Element, len(t.elements)) + copy(tt.elements, t.elements) + } + return tt +} diff --git a/common/serialize/log/table_test.go b/common/serialize/log/table_test.go new file mode 100644 index 000000000..3de169db8 --- /dev/null +++ b/common/serialize/log/table_test.go @@ -0,0 +1,562 @@ +// Copyright 2012-2013 Apcera Inc. All rights reserved. +package termtables + +import "testing" + +func DisplayFailedOutput(actual, expected string) string { + return "Output didn't match expected\n\n" + + "Actual:\n\n" + + actual + "\n" + + "Expected:\n\n" + + expected +} + +func checkRendersTo(t *testing.T, table *Table, expected string) { + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestCreateTable(t *testing.T) { + expected := "" + + "+-----------+-------+\n" + + "| Name | Value |\n" + + "+-----------+-------+\n" + + "| hey | you |\n" + + "| ken | 1234 |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "| escaping | rox%% |\n" + + "+-----------+-------+\n" + + table := CreateTable() + + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + table.AddRow("escaping", "rox%%") + + checkRendersTo(t, table, expected) +} + +func TestStyleResets(t *testing.T) { + expected := "" + + "+-----------+-------+\n" + + "| Name | Value |\n" + + "+-----------+-------+\n" + + "| hey | you |\n" + + "| ken | 1234 |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------+-------+\n" + + table := CreateTable() + table.UTF8Box() + table.Style.setAsciiBoxStyle() + + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + checkRendersTo(t, table, expected) +} + +func TestTableWithHeader(t *testing.T) { + expected := "" + + "+-------------------+\n" + + "| Example |\n" + + "+-----------+-------+\n" + + "| Name | Value |\n" + + "+-----------+-------+\n" + + "| hey | you |\n" + + "| ken | 1234 |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------+-------+\n" + + table := CreateTable() + + table.AddTitle("Example") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + checkRendersTo(t, table, expected) +} + +// TestTableWithHeaderMultipleTimes ensures that printing a table with headers +// multiple times continues to render correctly. +func TestTableWithHeaderMultipleTimes(t *testing.T) { + expected := "" + + "+-------------------+\n" + + "| Example |\n" + + "+-----------+-------+\n" + + "| Name | Value |\n" + + "+-----------+-------+\n" + + "| hey | you |\n" + + "| ken | 1234 |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------+-------+\n" + + table := CreateTable() + + table.AddTitle("Example") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + checkRendersTo(t, table, expected) + checkRendersTo(t, table, expected) +} + +func TestTableTitleWidthAdjusts(t *testing.T) { + expected := "" + + "+---------------------------+\n" + + "| Example My Foo Bar'd Test |\n" + + "+-----------+---------------+\n" + + "| Name | Value |\n" + + "+-----------+---------------+\n" + + "| hey | you |\n" + + "| ken | 1234 |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------+---------------+\n" + + table := CreateTable() + + table.AddTitle("Example My Foo Bar'd Test") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + checkRendersTo(t, table, expected) +} + +func TestTableHeaderWidthAdjusts(t *testing.T) { + expected := "" + + "+---------------+---------------------+\n" + + "| Slightly Long | More than 2 columns |\n" + + "+---------------+---------------------+\n" + + "| a | b |\n" + + "+---------------+---------------------+\n" + + table := CreateTable() + + table.AddHeaders("Slightly Long", "More than 2 columns") + table.AddRow("a", "b") + + checkRendersTo(t, table, expected) +} + +func TestTableWithNoHeaders(t *testing.T) { + expected := "" + + "+-----------+------+\n" + + "| hey | you |\n" + + "| ken | 1234 |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------+------+\n" + + table := CreateTable() + + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + checkRendersTo(t, table, expected) +} + +func TestTableUnicodeWidths(t *testing.T) { + expected := "" + + "+-----------+------+\n" + + "| Name | Cost |\n" + + "+-----------+------+\n" + + "| Currency | ¤10 |\n" + + "| US Dollar | $30 |\n" + + "| Euro | €27 |\n" + + "| Thai | ฿70 |\n" + + "+-----------+------+\n" + + table := CreateTable() + table.AddHeaders("Name", "Cost") + table.AddRow("Currency", "¤10") + table.AddRow("US Dollar", "$30") + table.AddRow("Euro", "€27") + table.AddRow("Thai", "฿70") + + checkRendersTo(t, table, expected) +} + +func TestTableInUTF8(t *testing.T) { + expected := "" + + "╭───────────────────╮\n" + + "│ Example │\n" + + "├───────────┬───────┤\n" + + "│ Name │ Value │\n" + + "├───────────┼───────┤\n" + + "│ hey │ you │\n" + + "│ ken │ 1234 │\n" + + "│ derek │ 3.14 │\n" + + "│ derek too │ 3.15 │\n" + + "│ escaping │ rox%% │\n" + + "╰───────────┴───────╯\n" + + table := CreateTable() + table.UTF8Box() + + table.AddTitle("Example") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("ken", 1234) + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + table.AddRow("escaping", "rox%%") + + checkRendersTo(t, table, expected) +} + +func TestTableUnicodeUTF8AndSGR(t *testing.T) { + // at present, this mostly just tests that alignment still works + expected := "" + + "╭───────────────────────╮\n" + + "│ \033[1mFanciness\033[0m │\n" + + "├──────────┬────────────┤\n" + + "│ \033[31mred\033[0m │ \033[32mgreen\033[0m │\n" + + "├──────────┼────────────┤\n" + + "│ plain │ text │\n" + + "│ Καλημέρα │ κόσμε │\n" + + "│ \033[1mvery\033[0m │ \033[4munderlined\033[0m │\n" + + "│ a\033[1mb\033[0mc │ \033[45mmagenta\033[0m │\n" + + "│ \033[31m→\033[0m │ \033[32m←\033[0m │\n" + + "╰──────────┴────────────╯\n" + + sgred := func(in string, sgrPm string) string { + return "\033[" + sgrPm + "m" + in + "\033[0m" + } + bold := func(in string) string { return sgred(in, "1") } + + table := CreateTable() + table.UTF8Box() + + table.AddTitle(bold("Fanciness")) + table.AddHeaders(sgred("red", "31"), sgred("green", "32")) + table.AddRow("plain", "text") + table.AddRow("Καλημέρα", "κόσμε") // from http://plan9.bell-labs.com/sys/doc/utf.html + table.AddRow(bold("very"), sgred("underlined", "4")) + table.AddRow("a"+bold("b")+"c", sgred("magenta", "45")) + table.AddRow(sgred("→", "31"), sgred("←", "32")) + // TODO: in future, if we start detecting presence of SGR sequences, we + // should ensure that the SGR reset is done at the end of the cell content, + // so that SGR doesn't "bleed across" (cells or rows). We would then add + // tests for that here. + // + // Of course, at that point, we'd also want to support automatic HTML + // styling conversion too, so would need a test for that also. + + checkRendersTo(t, table, expected) +} + +func TestTableInMarkdown(t *testing.T) { + expected := "" + + "Table: Example\n\n" + + "| Name | Value |\n" + + "| ----- | ----- |\n" + + "| hey | you |\n" + + "| a | b | esc |\n" + + "| esc | rox%% |\n" + + table := CreateTable() + table.SetModeMarkdown() + + table.AddTitle("Example") + table.AddHeaders("Name", "Value") + table.AddRow("hey", "you") + table.AddRow("a | b", "esc") + table.AddRow("esc", "rox%%") + + checkRendersTo(t, table, expected) +} + +func TestTitleUnicodeWidths(t *testing.T) { + expected := "" + + "+-------+\n" + + "| ← 5 → |\n" + + "+---+---+\n" + + "| a | b |\n" + + "| c | d |\n" + + "| e | 3 |\n" + + "+---+---+\n" + + // minimum width for a table of two columns is 9 characters, given + // one space of padding, and non-empty tables. + + table := CreateTable() + + // We have 4 characters down for left and right columns and padding, so + // a width of 5 for us should match the minimum per the columns + + // 5 characters; each arrow is three octets in UTF-8, giving 9 bytes + // so, same in character-count-width, longer in bytes + table.AddTitle("← 5 →") + + // a single character per cell, here; use ASCII characters + table.AddRow("a", "b") + table.AddRow("c", "d") + table.AddRow("e", 3) + + checkRendersTo(t, table, expected) +} + +// We identified two error conditions wherein length wrapping would not correctly +// wrap width when, for instance, in a two-column table, the longest row in the +// right-hand column was not the same as the longest row in the left-hand column. +// This tests that we correctly accumulate the maximum width across all rows of +// the termtable and adjust width accordingly. +func TestTableWidthHandling(t *testing.T) { + expected := "" + + "+-----------------------------------------+\n" + + "| Example... to Fix My Test |\n" + + "+-----------------+-----------------------+\n" + + "| hey foo bar baz | you |\n" + + "| ken | you should write code |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------------+-----------------------+\n" + + table := CreateTable() + + table.AddTitle("Example... to Fix My Test") + table.AddRow("hey foo bar baz", "you") + table.AddRow("ken", "you should write code") + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } + +} + +func TestTableWidthHandling_SecondErrorCondition(t *testing.T) { + expected := "" + + "+----------------------------------------+\n" + + "| Example... to Fix My Test |\n" + + "+-----------------+----------------------+\n" + + "| hey foo bar baz | you |\n" + + "| ken | you should sell cod! |\n" + + "| derek | 3.14 |\n" + + "| derek too | 3.15 |\n" + + "+-----------------+----------------------+\n" + + table := CreateTable() + + table.AddTitle("Example... to Fix My Test") + table.AddRow("hey foo bar baz", "you") + table.AddRow("ken", "you should sell cod!") + table.AddRow("derek", 3.14) + table.AddRow("derek too", 3.1456788) + + output := table.Render() + if output != expected { + t.Fatal(DisplayFailedOutput(output, expected)) + } +} + +func TestTableAlignPostsetting(t *testing.T) { + expected := "" + + "+-----------+-------+----------+\n"+ + "| Name | Value | Value 2 |\n"+ + "+-----------+-------+----------+\n"+ + "| hey | you | man |\n"+ + "| ken | 1234 | 4321 |\n"+ + "| derek | 3.14 | bob |\n"+ + "| derek too | 3.15 | long bob |\n"+ + "| escaping | rox%% | :) |\n"+ + "+-----------+-------+----------+\n" + + table := CreateTable() + + table.AddHeaders("Name", "Value", "Value 2") + table.AddRow("hey", "you", "man") + table.AddRow("ken", 1234, 4321) + table.AddRow("derek", 3.14, "bob") + table.AddRow("derek too", 3.1456788, "long bob") + table.AddRow("escaping", "rox%%", ":)") + + table.SetAlign(AlignRight, 2, 3) + + checkRendersTo(t, table, expected) +} + +func TestTableMissingCells(t *testing.T) { + expected := "" + + "+----------+---------+---------+\n" + + "| Name | Value 1 | Value 2 |\n" + + "+----------+---------+---------+\n" + + "| hey | you | person |\n" + + "| ken | 1234 |\n" + + "| escaping | rox%s%% |\n" + + "+----------+---------+---------+\n" + // FIXME: missing extra cells there + + table := CreateTable() + + table.AddHeaders("Name", "Value 1", "Value 2") + table.AddRow("hey", "you", "person") + table.AddRow("ken", 1234) + table.AddRow("escaping", "rox%s%%") + + checkRendersTo(t, table, expected) +} + +// We don't yet support combining characters, double-width characters or +// anything to do with estimating a tty-style "character width" for what in +// Unicode is a grapheme cluster. This disabled test shows what we want +// to support, but don't yet. +func TestTableWithCombiningChars(t *testing.T) { + expected := "" + + "+------+---+\n" + + "| noel | 1 |\n" + + "| noël | 2 |\n" + + "| noël | 3 |\n" + + "+------+---+\n" + + table := CreateTable() + + table.AddRow("noel", "1") + table.AddRow("noe\u0308l", "2") // LATIN SMALL LETTER E + COMBINING DIAERESIS + table.AddRow("noël", "3") // Hex EB; LATIN SMALL LETTER E WITH DIAERESIS + + checkRendersTo(t, table, expected) +} + +// another unicode length issue +func TestTableWithFullwidthChars(t *testing.T) { + expected := "" + + "+----------+------------+\n" + + "| wide | not really |\n" + + "| ｗｉｄｅ | fullwidth |\n" + + "+----------+------------+\n" + + table := CreateTable() + table.AddRow("wide", "not really") + table.AddRow("ｗｉｄｅ", "fullwidth") // FULLWIDTH LATIN SMALL LETTER + + checkRendersTo(t, table, expected) +} + +// Tests CJK characters using examples given in issue #33. The examples may not +// look like they line up but you can visually confirm its accuracy with a +// fmt.Print. +func TestCJKChars(t *testing.T) { + expected := "" + + "+-------+---------+----------+\n" + + "| KeyID | ValueID | ValueCN |\n" + + "+-------+---------+----------+\n" + + "| 8 | 51 | 精钢 |\n" + + "| 8 | 52 | 鳄鱼皮 |\n" + + "| 8 | 53 | 镀金皮带 |\n" + + "| 8 | 54 | 精钢 |\n" + + "+-------+---------+----------+\n" + + table := CreateTable() + table.AddHeaders("KeyID", "ValueID", "ValueCN") + table.AddRow("8", 51, "精钢") + table.AddRow("8", 52, "鳄鱼皮") + table.AddRow("8", 53, "镀金皮带") + table.AddRow("8", 54, "精钢") + checkRendersTo(t, table, expected) + + expected2 := "" + + "+--------------------+----------------------+\n" + + "| field | value |\n" + + "+--------------------+----------------------+\n" + + "| GoodsPropertyKeyID | 9 |\n" + + "| MerchantAccountID | 0 |\n" + + "| GoodsCategoryCode | 100001 |\n" + + "| NameCN | 机芯类型 |\n" + + "| NameJP | ムーブメントのタイプ |\n" + + "+--------------------+----------------------+\n" + table = CreateTable() + table.AddHeaders("field", "value") + table.AddRow("GoodsPropertyKeyID", 9) + table.AddRow("MerchantAccountID", 0) + table.AddRow("GoodsCategoryCode", 100001) + table.AddRow("NameCN", "机芯类型") + table.AddRow("NameJP", "ムーブメントのタイプ") + checkRendersTo(t, table, expected2) +} + +func TestTableMultipleAddHeader(t *testing.T) { + expected := "" + + "+--------------+--------+-------+\n" + + "| First column | Second | Third |\n" + + "+--------------+--------+-------+\n" + + "| 2 | 3 | 5 |\n" + + "+--------------+--------+-------+\n" + + table := CreateTable() + table.AddHeaders("First column", "Second") + table.AddHeaders("Third") + table.AddRow(2, 3, 5) + + checkRendersTo(t, table, expected) +} + +func createTestTable() *Table { + table := CreateTable() + header := []interface{}{} + for i := 0; i < 50; i++ { + header = append(header, "First Column") + } + table.AddHeaders(header...) + for i := 0; i < 3000; i++ { + row := []interface{}{} + for i := 0; i < 50; i++ { + row = append(row, "First row value") + } + table.AddRow(row...) + } + return table +} + +func BenchmarkTableRenderTerminal(b *testing.B) { + table := createTestTable() + table.SetModeTerminal() + b.ResetTimer() + for i := 0; i < b.N; i++ { + table.Render() + } +} + +func BenchmarkTableRenderMarkdown(b *testing.B) { + table := createTestTable() + table.SetModeMarkdown() + b.ResetTimer() + for i := 0; i < b.N; i++ { + table.Render() + } +} + +func BenchmarkTableRenderHTML(b *testing.B) { + table := createTestTable() + table.SetModeHTML() + b.ResetTimer() + for i := 0; i < b.N; i++ { + table.Render() + } +} diff --git a/common/serialize/log/term/env.go b/common/serialize/log/term/env.go new file mode 100644 index 000000000..2bcbea750 --- /dev/null +++ b/common/serialize/log/term/env.go @@ -0,0 +1,43 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +package term + +import ( + "os" + "strconv" +) + +// GetEnvWindowSize returns the window Size, as determined by process +// environment; if either LINES or COLUMNS is present, and whichever is +// present is also numeric, the Size will be non-nil. If Size is nil, +// there's insufficient data in environ. If one entry is 0, that means +// that the environment does not include that data. If a value is +// negative, we treat that as an error. +func GetEnvWindowSize() *Size { + lines := os.Getenv("LINES") + columns := os.Getenv("COLUMNS") + if lines == "" && columns == "" { + return nil + } + + nLines := 0 + nColumns := 0 + var err error + if lines != "" { + nLines, err = strconv.Atoi(lines) + if err != nil || nLines < 0 { + return nil + } + } + if columns != "" { + nColumns, err = strconv.Atoi(columns) + if err != nil || nColumns < 0 { + return nil + } + } + + return &Size{ + Lines: nLines, + Columns: nColumns, + } +} diff --git a/common/serialize/log/term/getsize.go b/common/serialize/log/term/getsize.go new file mode 100644 index 000000000..3521ef913 --- /dev/null +++ b/common/serialize/log/term/getsize.go @@ -0,0 +1,54 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +package term + +import ( + "os" +) + +// Size is the size of a terminal, expressed in character cells, as Lines and +// Columns. This might come from environment variables or OS-dependent +// resources. +type Size struct { + Lines int + Columns int +} + +// GetSize will return the terminal window size. +// +// We prefer environ $LINES/$COLUMNS, then fall back to tty-held information. +// We do not support use of termcap/terminfo to derive default size information. +func GetSize() (*Size, error) { + envSize := GetEnvWindowSize() + if envSize != nil && envSize.Lines != 0 && envSize.Columns != 0 { + return envSize, nil + } + + fh, err := os.Open("/dev/tty") + if err != nil { + // no tty, no point continuing; we only let the environ + // avoid an error in this case because if someone has faked + // up an environ with LINES/COLUMNS _both_ set, we should let + // them + return nil, err + } + + size, err := GetTerminalWindowSize(fh) + if err != nil { + if envSize != nil { + return envSize, nil + } + return nil, err + } + if envSize == nil { + return size, err + } + + if envSize.Lines == 0 { + envSize.Lines = size.Lines + } + if envSize.Columns == 0 { + envSize.Columns = size.Columns + } + return envSize, nil +} diff --git a/common/serialize/log/term/sizes_unix.go b/common/serialize/log/term/sizes_unix.go new file mode 100644 index 000000000..c7ad9ded4 --- /dev/null +++ b/common/serialize/log/term/sizes_unix.go @@ -0,0 +1,35 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +// +build !windows + +package term + +import ( + "errors" + "os" + "syscall" + "unsafe" +) + +// ErrGetWinsizeFailed indicates that the system call to extract the size of +// a Unix tty from the kernel failed. +var ErrGetWinsizeFailed = errors.New("term: syscall.TIOCGWINSZ failed") + +// GetTerminalWindowSize returns the terminal size maintained by the kernel +// for a Unix TTY, passed in as an *os.File. This information can be seen +// with the stty(1) command, and changes in size (eg, terminal emulator +// resized) should trigger a SIGWINCH signal delivery to the foreground process +// group at the time of the change, so a long-running process might reasonably +// watch for SIGWINCH and arrange to re-fetch the size when that happens. +func GetTerminalWindowSize(file *os.File) (*Size, error) { + // Based on source from from golang.org/x/crypto/ssh/terminal/util.go + var dimensions [4]uint16 + if _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, file.Fd(), uintptr(syscall.TIOCGWINSZ), uintptr(unsafe.Pointer(&dimensions)), 0, 0, 0); err != 0 { + return nil, err + } + + return &Size{ + Lines: int(dimensions[0]), + Columns: int(dimensions[1]), + }, nil +} diff --git a/common/serialize/log/term/sizes_windows.go b/common/serialize/log/term/sizes_windows.go new file mode 100644 index 000000000..0ca5400b9 --- /dev/null +++ b/common/serialize/log/term/sizes_windows.go @@ -0,0 +1,57 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +// +build windows + +package term + +// Used when we have no other source for getting platform-specific information +// about the terminal sizes available. + +import ( + "os" + "syscall" + "unsafe" +) + +// Based on source from from golang.org/x/crypto/ssh/terminal/util_windows.go +var ( + kernel32 = syscall.NewLazyDLL("kernel32.dll") + + procGetConsoleScreenBufferInfo = kernel32.NewProc("GetConsoleScreenBufferInfo") +) + +type ( + short int16 + word uint16 + + coord struct { + x short + y short + } + smallRect struct { + left short + top short + right short + bottom short + } + consoleScreenBufferInfo struct { + size coord + cursorPosition coord + attributes word + window smallRect + maximumWindowSize coord + } +) + +// GetTerminalWindowSize returns the width and height of a terminal in Windows. +func GetTerminalWindowSize(file *os.File) (*Size, error) { + var info consoleScreenBufferInfo + _, _, e := syscall.Syscall(procGetConsoleScreenBufferInfo.Addr(), 2, file.Fd(), uintptr(unsafe.Pointer(&info)), 0) + if e != 0 { + return nil, error(e) + } + return &Size{ + Lines: int(info.size.y), + Columns: int(info.size.x), + }, nil +} diff --git a/common/serialize/log/term/wrapper.go b/common/serialize/log/term/wrapper.go new file mode 100644 index 000000000..4a7766e61 --- /dev/null +++ b/common/serialize/log/term/wrapper.go @@ -0,0 +1,23 @@ +// Copyright 2013 Apcera Inc. All rights reserved. + +//go:build ignore +// +build ignore + +package main + +import ( + "fmt" + "os" + + "github.com/apcera/termtables/term" +) + +func main() { + size, err := term.GetSize() + if err != nil { + fmt.Fprintf(os.Stderr, "failed: %s\n", err) + os.Exit(1) + } + + fmt.Printf("Lines %d Columns %d\n", size.Lines, size.Columns) +} diff --git a/common/serialize/bitset/bitset_test.go b/common/serialize/test_test.go similarity index 69% rename from common/serialize/bitset/bitset_test.go rename to common/serialize/test_test.go index 64abcb732..e9c784048 100644 --- a/common/serialize/bitset/bitset_test.go +++ b/common/serialize/test_test.go @@ -1,13 +1,14 @@ -package bitset +package serialize import ( "fmt" "testing" + "github.com/apcera/termtables" "github.com/pointernil/bitset32" ) -func Test_teset(t *testing.T) { +func TestThree(t *testing.T) { fmt.Printf("! \n") var b bitset32.BitSet32 var a bitset32.BitSet32 @@ -35,3 +36,14 @@ func Test_teset(t *testing.T) { } fmt.Println(b.Bytes()) } + +func TestInit(t *testing.T) { + table := termtables.CreateTable() + + table.AddHeaders("Name", "Age") + table.AddRow("John", "30") + table.AddRow("Sam", 18) + table.AddRow("Julie", 20.14) + + fmt.Println(table.Render()) +} diff --git a/go.work b/go.work index 8d78c7ef5..619482b11 100644 --- a/go.work +++ b/go.work @@ -5,6 +5,8 @@ use ( ./common/contrib/drivers/mysql ./common/contrib/files/local ./common/cool + ./common/serialize/bitset + ./common/serialize/log ./common/serialize/sturc ./common/serialize/xml ./logic