"refactor(common): 重构序列化工具包，将serialize重命名为utils并添加bitset组件"

common/utils/sturc/.travis.yml

@@ -0,0 +1,10 @@
+language: go
+sudo: false
+
+script: go test -v
+
+go:
+  - 1.3
+  - 1.12
+  - 1.13
+  - tip

common/utils/sturc/LICENSE

@@ -0,0 +1,19 @@
+Copyright (c) 2015 Ryan Hileman
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.

common/utils/sturc/README.md

@@ -0,0 +1,104 @@
+[![Build Status](https://travis-ci.org/lunixbochs/struc.svg?branch=master)](https://travis-ci.org/lunixbochs/struc) [![GoDoc](https://godoc.org/github.com/lunixbochs/struc?status.svg)](https://godoc.org/github.com/lunixbochs/struc)
+
+struc
+====
+
+Struc exists to pack and unpack C-style structures from bytes, which is useful for binary files and network protocols. It could be considered an alternative to `encoding/binary`, which requires massive boilerplate for some similar operations.
+
+Take a look at an [example comparing `struc` and `encoding/binary`](https://bochs.info/p/cxvm9)
+
+Struc considers usability first. That said, it does cache reflection data and aims to be competitive with `encoding/binary` struct packing in every way, including performance.
+
+Example struct
+----
+
+```Go
+type Example struct {
+    Var   int `struc:"int32,sizeof=Str"`
+    Str   string
+    Weird []byte `struc:"[8]int64"`
+     Weird []byte `struc:"[32]byte"`
+    Var   []int `struc:"[]int32,little"`
+}
+```
+
+Struct tag format
+----
+
+ - ```Var []int `struc:"[]int32,little,sizeof=StringField"` ``` will pack Var as a slice of little-endian int32, and link it as the size of `StringField`.
+ - `sizeof=`: Indicates this field is a number used to track the length of a another field. `sizeof` fields are automatically updated on `Pack()` based on the current length of the tracked field, and are used to size the target field during `Unpack()`.
+ - Bare values will be parsed as type and endianness.
+
+Endian formats
+----
+
+ - `big` (default)
+ - `little`
+
+Recognized types
+----
+
+ - `pad` - this type ignores field contents and is backed by a `[length]byte` containing nulls
+ - `bool`
+ - `byte`
+ - `int8`, `uint8`
+ - `int16`, `uint16`
+ - `int32`, `uint32`
+ - `int64`, `uint64`
+ - `float32`
+ - `float64`
+
+Types can be indicated as arrays/slices using `[]` syntax. Example: `[]int64`, `[8]int32`.
+
+Bare slice types (those with no `[size]`) must have a linked `Sizeof` field.
+
+Private fields are ignored when packing and unpacking.
+
+Example code
+----
+
+```Go
+package main
+
+import (
+    "bytes"
+    "github.com/lunixbochs/struc"
+)
+
+type Example struct {
+    A int `struc:"big"`
+
+    // B will be encoded/decoded as a 16-bit int (a "short")
+    // but is stored as a native int in the struct
+    B int `struc:"int16"`
+
+    // the sizeof key links a buffer's size to any int field
+    Size int `struc:"int8,little,sizeof=Str"`
+    Str  string
+
+    // you can get freaky if you want
+    Str2 string `struc:"[5]int64"`
+}
+
+func main() {
+    var buf bytes.Buffer
+    t := &Example{1, 2, 0, "test", "test2"}
+    err := struc.Pack(&buf, t)
+    o := &Example{}
+    err = struc.Unpack(&buf, o)
+}
+```
+
+Benchmark
+----
+
+`BenchmarkEncode` uses struc. `Stdlib` benchmarks use equivalent `encoding/binary` code. `Manual` encodes without any reflection, and should be considered an upper bound on performance (which generated code based on struc definitions should be able to achieve).
+
+```
+BenchmarkEncode        1000000   1265 ns/op
+BenchmarkStdlibEncode  1000000   1855 ns/op
+BenchmarkManualEncode  5000000    284 ns/op
+BenchmarkDecode        1000000   1259 ns/op
+BenchmarkStdlibDecode  1000000   1656 ns/op
+BenchmarkManualDecode  20000000  89.0 ns/op
+```

common/utils/sturc/bench_test.go

@@ -0,0 +1,203 @@
+package struc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"testing"
+)
+
+type BenchExample struct {
+	Test    [5]byte
+	A       int32
+	B, C, D int16
+	Test2   [4]byte
+	Length  int32
+}
+
+func BenchmarkArrayEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var buf bytes.Buffer
+		if err := Pack(&buf, arrayReference); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkSliceEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var buf bytes.Buffer
+		if err := Pack(&buf, sliceReference); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkArrayDecode(b *testing.B) {
+	var out ExampleArray
+	for i := 0; i < b.N; i++ {
+		buf := bytes.NewBuffer(arraySliceReferenceBytes)
+		if err := Unpack(buf, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkSliceDecode(b *testing.B) {
+	var out ExampleSlice
+	for i := 0; i < b.N; i++ {
+		buf := bytes.NewBuffer(arraySliceReferenceBytes)
+		if err := Unpack(buf, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+type BenchStrucExample struct {
+	Test    [5]byte `struc:"[5]byte"`
+	A       int     `struc:"int32"`
+	B, C, D int     `struc:"int16"`
+	Test2   [4]byte `struc:"[4]byte"`
+	Length  int     `struc:"int32,sizeof=Data"`
+	Data    []byte
+}
+
+var benchRef = &BenchExample{
+	[5]byte{1, 2, 3, 4, 5},
+	1, 2, 3, 4,
+	[4]byte{1, 2, 3, 4},
+	8,
+}
+
+var eightBytes = []byte("8bytestr")
+
+var benchStrucRef = &BenchStrucExample{
+	[5]byte{1, 2, 3, 4, 5},
+	1, 2, 3, 4,
+	[4]byte{1, 2, 3, 4},
+	8, eightBytes,
+}
+
+func BenchmarkEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var buf bytes.Buffer
+		err := Pack(&buf, benchStrucRef)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkStdlibEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var buf bytes.Buffer
+		err := binary.Write(&buf, binary.BigEndian, benchRef)
+		if err != nil {
+			b.Fatal(err)
+		}
+		_, err = buf.Write(eightBytes)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkManualEncode(b *testing.B) {
+	order := binary.BigEndian
+	s := benchStrucRef
+	for i := 0; i < b.N; i++ {
+		var buf bytes.Buffer
+		tmp := make([]byte, 29)
+		copy(tmp[0:5], s.Test[:])
+		order.PutUint32(tmp[5:9], uint32(s.A))
+		order.PutUint16(tmp[9:11], uint16(s.B))
+		order.PutUint16(tmp[11:13], uint16(s.C))
+		order.PutUint16(tmp[13:15], uint16(s.D))
+		copy(tmp[15:19], s.Test2[:])
+		order.PutUint32(tmp[19:23], uint32(s.Length))
+		copy(tmp[23:], s.Data)
+		_, err := buf.Write(tmp)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkDecode(b *testing.B) {
+	var out BenchStrucExample
+	var buf bytes.Buffer
+	if err := Pack(&buf, benchStrucRef); err != nil {
+		b.Fatal(err)
+	}
+	bufBytes := buf.Bytes()
+	for i := 0; i < b.N; i++ {
+		buf := bytes.NewReader(bufBytes)
+		err := Unpack(buf, &out)
+		if err != nil {
+			b.Fatal(err)
+		}
+		out.Data = nil
+	}
+}
+
+func BenchmarkStdlibDecode(b *testing.B) {
+	var out BenchExample
+	var buf bytes.Buffer
+	binary.Write(&buf, binary.BigEndian, *benchRef)
+	_, err := buf.Write(eightBytes)
+	if err != nil {
+		b.Fatal(err)
+	}
+	bufBytes := buf.Bytes()
+	for i := 0; i < b.N; i++ {
+		buf := bytes.NewReader(bufBytes)
+		err := binary.Read(buf, binary.BigEndian, &out)
+		if err != nil {
+			b.Fatal(err)
+		}
+		tmp := make([]byte, out.Length)
+		_, err = buf.Read(tmp)
+		if err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkManualDecode(b *testing.B) {
+	var o BenchStrucExample
+	var buf bytes.Buffer
+	if err := Pack(&buf, benchStrucRef); err != nil {
+		b.Fatal(err)
+	}
+	tmp := buf.Bytes()
+	order := binary.BigEndian
+	for i := 0; i < b.N; i++ {
+		copy(o.Test[:], tmp[0:5])
+		o.A = int(order.Uint32(tmp[5:9]))
+		o.B = int(order.Uint16(tmp[9:11]))
+		o.C = int(order.Uint16(tmp[11:13]))
+		o.D = int(order.Uint16(tmp[13:15]))
+		copy(o.Test2[:], tmp[15:19])
+		o.Length = int(order.Uint32(tmp[19:23]))
+		o.Data = make([]byte, o.Length)
+		copy(o.Data, tmp[23:])
+	}
+}
+
+func BenchmarkFullEncode(b *testing.B) {
+	for i := 0; i < b.N; i++ {
+		var buf bytes.Buffer
+		if err := Pack(&buf, reference); err != nil {
+			b.Fatal(err)
+		}
+	}
+}
+
+func BenchmarkFullDecode(b *testing.B) {
+	var out Example
+	for i := 0; i < b.N; i++ {
+		buf := bytes.NewBuffer(referenceBytes)
+		if err := Unpack(buf, &out); err != nil {
+			b.Fatal(err)
+		}
+	}
+}

common/utils/sturc/binary.go

@@ -0,0 +1,52 @@
+package struc
+
+import (
+	"encoding/binary"
+	"io"
+	"reflect"
+)
+
+type byteWriter struct {
+	buf []byte
+	pos int
+}
+
+func (b byteWriter) Write(p []byte) (int, error) {
+	capacity := len(b.buf) - b.pos
+	if capacity < len(p) {
+		p = p[:capacity]
+	}
+	if len(p) > 0 {
+		copy(b.buf[b.pos:], p)
+		b.pos += len(p)
+	}
+	return len(p), nil
+}
+
+type binaryFallback reflect.Value
+
+func (b binaryFallback) String() string {
+	return b.String()
+}
+
+func (b binaryFallback) Sizeof(val reflect.Value, options *Options) int {
+	return binary.Size(val.Interface())
+}
+
+func (b binaryFallback) Pack(buf []byte, val reflect.Value, options *Options) (int, error) {
+	tmp := byteWriter{buf: buf}
+	var order binary.ByteOrder = binary.BigEndian
+	if options.Order != nil {
+		order = options.Order
+	}
+	err := binary.Write(tmp, order, val.Interface())
+	return tmp.pos, err
+}
+
+func (b binaryFallback) Unpack(r io.Reader, val reflect.Value, options *Options) error {
+	var order binary.ByteOrder = binary.BigEndian
+	if options.Order != nil {
+		order = options.Order
+	}
+	return binary.Read(r, order, val.Interface())
+}

common/utils/sturc/custom.go

@@ -0,0 +1,33 @@
+package struc
+
+import (
+	"io"
+	"reflect"
+)
+
+type Custom interface {
+	Pack(p []byte, opt *Options) (int, error)
+	Unpack(r io.Reader, length int, opt *Options) error
+	Size(opt *Options) int
+	String() string
+}
+
+type customFallback struct {
+	custom Custom
+}
+
+func (c customFallback) Pack(p []byte, val reflect.Value, opt *Options) (int, error) {
+	return c.custom.Pack(p, opt)
+}
+
+func (c customFallback) Unpack(r io.Reader, val reflect.Value, opt *Options) error {
+	return c.custom.Unpack(r, 1, opt)
+}
+
+func (c customFallback) Sizeof(val reflect.Value, opt *Options) int {
+	return c.custom.Size(opt)
+}
+
+func (c customFallback) String() string {
+	return c.custom.String()
+}

common/utils/sturc/custom_float16.go

@@ -0,0 +1,78 @@
+package struc
+
+import (
+	"encoding/binary"
+	"io"
+	"math"
+	"strconv"
+)
+
+type Float16 float64
+
+func (f *Float16) Pack(p []byte, opt *Options) (int, error) {
+	order := opt.Order
+	if order == nil {
+		order = binary.BigEndian
+	}
+	sign := uint16(0)
+	if *f < 0 {
+		sign = 1
+	}
+	var frac, exp uint16
+	if math.IsInf(float64(*f), 0) {
+		exp = 0x1f
+		frac = 0
+	} else if math.IsNaN(float64(*f)) {
+		exp = 0x1f
+		frac = 1
+	} else {
+		bits := math.Float64bits(float64(*f))
+		exp64 := (bits >> 52) & 0x7ff
+		if exp64 != 0 {
+			exp = uint16((exp64 - 1023 + 15) & 0x1f)
+		}
+		frac = uint16((bits >> 42) & 0x3ff)
+	}
+	var out uint16
+	out |= sign << 15
+	out |= exp << 10
+	out |= frac & 0x3ff
+	order.PutUint16(p, out)
+	return 2, nil
+}
+func (f *Float16) Unpack(r io.Reader, length int, opt *Options) error {
+	order := opt.Order
+	if order == nil {
+		order = binary.BigEndian
+	}
+	var tmp [2]byte
+	if _, err := r.Read(tmp[:]); err != nil {
+		return err
+	}
+	val := order.Uint16(tmp[:2])
+	sign := (val >> 15) & 1
+	exp := int16((val >> 10) & 0x1f)
+	frac := val & 0x3ff
+	if exp == 0x1f {
+		if frac != 0 {
+			*f = Float16(math.NaN())
+		} else {
+			*f = Float16(math.Inf(int(sign)*-2 + 1))
+		}
+	} else {
+		var bits uint64
+		bits |= uint64(sign) << 63
+		bits |= uint64(frac) << 42
+		if exp > 0 {
+			bits |= uint64(exp-15+1023) << 52
+		}
+		*f = Float16(math.Float64frombits(bits))
+	}
+	return nil
+}
+func (f *Float16) Size(opt *Options) int {
+	return 2
+}
+func (f *Float16) String() string {
+	return strconv.FormatFloat(float64(*f), 'g', -1, 32)
+}

common/utils/sturc/custom_float16_test.go

@@ -0,0 +1,56 @@
+package struc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"strconv"
+	"strings"
+	"testing"
+)
+
+func TestFloat16(t *testing.T) {
+	// test cases from https://en.wikipedia.org/wiki/Half-precision_floating-point_format#Half_precision_examples
+	tests := []struct {
+		B string
+		F float64
+	}{
+		//s expnt significand
+		{"0 01111 0000000000", 1},
+		{"0 01111 0000000001", 1.0009765625},
+		{"1 10000 0000000000", -2},
+		{"0 11110 1111111111", 65504},
+		// {"0 00001 0000000000", 0.0000610352},
+		// {"0 00000 1111111111", 0.0000609756},
+		// {"0 00000 0000000001", 0.0000000596046},
+		{"0 00000 0000000000", 0},
+		// {"1 00000 0000000000", -0},
+		{"0 11111 0000000000", math.Inf(1)},
+		{"1 11111 0000000000", math.Inf(-1)},
+		{"0 01101 0101010101", 0.333251953125},
+	}
+	for _, test := range tests {
+		var buf bytes.Buffer
+		f := Float16(test.F)
+		if err := Pack(&buf, &f); err != nil {
+			t.Error("pack failed:", err)
+			continue
+		}
+		bitval, _ := strconv.ParseUint(strings.Replace(test.B, " ", "", -1), 2, 16)
+		tmp := binary.BigEndian.Uint16(buf.Bytes())
+		if tmp != uint16(bitval) {
+			t.Errorf("incorrect pack: %s != %016b (%f)", test.B, tmp, test.F)
+			continue
+		}
+		var f2 Float16
+		if err := Unpack(&buf, &f2); err != nil {
+			t.Error("unpack failed:", err)
+			continue
+		}
+		// let sprintf deal with (im)precision for me here
+		if fmt.Sprintf("%f", f) != fmt.Sprintf("%f", f2) {
+			t.Errorf("incorrect unpack: %016b %f != %f", bitval, f, f2)
+		}
+	}
+}

common/utils/sturc/custom_test.go

@@ -0,0 +1,360 @@
+package struc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"io"
+	"reflect"
+	"strconv"
+	"testing"
+)
+
+// Custom Type
+type Int3 uint32
+
+// newInt3 returns a pointer to an Int3
+func newInt3(in int) *Int3 {
+	i := Int3(in)
+	return &i
+}
+
+type Int3Struct struct {
+	I Int3
+}
+
+func (i *Int3) Pack(p []byte, opt *Options) (int, error) {
+	var tmp [4]byte
+	binary.BigEndian.PutUint32(tmp[:], uint32(*i))
+	copy(p, tmp[1:])
+	return 3, nil
+}
+func (i *Int3) Unpack(r io.Reader, length int, opt *Options) error {
+	var tmp [4]byte
+	if _, err := r.Read(tmp[1:]); err != nil {
+		return err
+	}
+	*i = Int3(binary.BigEndian.Uint32(tmp[:]))
+	return nil
+}
+func (i *Int3) Size(opt *Options) int {
+	return 3
+}
+func (i *Int3) String() string {
+	return strconv.FormatUint(uint64(*i), 10)
+}
+
+// Array of custom type
+type ArrayInt3Struct struct {
+	I [2]Int3
+}
+
+// Custom type of array of standard type
+type DoubleUInt8 [2]uint8
+
+type DoubleUInt8Struct struct {
+	I DoubleUInt8
+}
+
+func (di *DoubleUInt8) Pack(p []byte, opt *Options) (int, error) {
+	for i, value := range *di {
+		p[i] = value
+	}
+
+	return 2, nil
+}
+
+func (di *DoubleUInt8) Unpack(r io.Reader, length int, opt *Options) error {
+	for i := 0; i < 2; i++ {
+		var value uint8
+		if err := binary.Read(r, binary.LittleEndian, &value); err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		di[i] = value
+	}
+	return nil
+}
+
+func (di *DoubleUInt8) Size(opt *Options) int {
+	return 2
+}
+
+func (di *DoubleUInt8) String() string {
+	panic("not implemented")
+}
+
+// Custom type of array of custom type
+type DoubleInt3 [2]Int3
+
+type DoubleInt3Struct struct {
+	D DoubleInt3
+}
+
+func (di *DoubleInt3) Pack(p []byte, opt *Options) (int, error) {
+	var out []byte
+	for _, value := range *di {
+		tmp := make([]byte, 3)
+		if _, err := value.Pack(tmp, opt); err != nil {
+			return 0, err
+		}
+		out = append(out, tmp...)
+	}
+	copy(p, out)
+
+	return 6, nil
+}
+
+func (di *DoubleInt3) Unpack(r io.Reader, length int, opt *Options) error {
+	for i := 0; i < 2; i++ {
+		di[i].Unpack(r, 0, opt)
+	}
+	return nil
+}
+
+func (di *DoubleInt3) Size(opt *Options) int {
+	return 6
+}
+
+func (di *DoubleInt3) String() string {
+	panic("not implemented")
+}
+
+// Custom type of slice of standard type
+// Slice of uint8, stored in a zero terminated list.
+type SliceUInt8 []uint8
+
+type SliceUInt8Struct struct {
+	I SliceUInt8
+	N uint8 // A field after to ensure the length is correct.
+}
+
+func (ia *SliceUInt8) Pack(p []byte, opt *Options) (int, error) {
+	for i, value := range *ia {
+		p[i] = value
+	}
+
+	return len(*ia) + 1, nil
+}
+
+func (ia *SliceUInt8) Unpack(r io.Reader, length int, opt *Options) error {
+	for {
+		var value uint8
+		if err := binary.Read(r, binary.LittleEndian, &value); err != nil {
+			if err == io.EOF {
+				return io.ErrUnexpectedEOF
+			}
+			return err
+		}
+		if value == 0 {
+			break
+		}
+		*ia = append(*ia, value)
+	}
+	return nil
+}
+
+func (ia *SliceUInt8) Size(opt *Options) int {
+	return len(*ia) + 1
+}
+
+func (ia *SliceUInt8) String() string {
+	panic("not implemented")
+}
+
+type ArrayOfUInt8Struct struct {
+	I [2]uint8
+}
+
+// Custom 4-character fixed string, similar to CHAR(4) in SQL.
+type Char4 string
+
+func (*Char4) Size(opt *Options) int {
+	return 4
+}
+
+func (c *Char4) Pack(p []byte, opt *Options) (int, error) {
+	buf := []byte(*c)
+	buf = append(buf, make([]byte, c.Size(nil)-len(buf))...)
+
+	copy(p, buf)
+	return len(buf), nil
+}
+
+func (c *Char4) Unpack(r io.Reader, length int, opt *Options) error {
+	buf := bytes.Buffer{}
+	if _, err := buf.ReadFrom(r); err != nil {
+		if err == io.EOF {
+			return io.ErrUnexpectedEOF
+		}
+		return err
+	}
+
+	*c = Char4(buf.String())
+	return nil
+}
+
+func (c *Char4) String() string {
+	return string(*c)
+}
+
+type Char4Struct struct {
+	C Char4
+}
+
+func TestCustomTypes(t *testing.T) {
+	testCases := []struct {
+		name        string
+		packObj     interface{}
+		emptyObj    interface{}
+		expectBytes []byte
+		skip        bool // Skip the test, because it fails.
+		// Switch to expectFail when possible:
+		// https://github.com/golang/go/issues/25951
+	}{
+		// Start tests with unimplemented non-custom types.
+		{
+			name:        "ArrayOfUInt8",
+			packObj:     [2]uint8{32, 64},
+			emptyObj:    [2]uint8{0, 0},
+			expectBytes: []byte{32, 64},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "PointerToArrayOfUInt8",
+			packObj:     &[2]uint8{32, 64},
+			emptyObj:    &[2]uint8{0, 0},
+			expectBytes: []byte{32, 64},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "ArrayOfUInt8Struct",
+			packObj:     &ArrayOfUInt8Struct{I: [2]uint8{32, 64}},
+			emptyObj:    &ArrayOfUInt8Struct{},
+			expectBytes: []byte{32, 64},
+		},
+		{
+			name:        "CustomType",
+			packObj:     newInt3(3),
+			emptyObj:    newInt3(0),
+			expectBytes: []byte{0, 0, 3},
+		},
+		{
+			name:        "CustomType-Big",
+			packObj:     newInt3(4000),
+			emptyObj:    newInt3(0),
+			expectBytes: []byte{0, 15, 160},
+		},
+		{
+			name:        "CustomTypeStruct",
+			packObj:     &Int3Struct{3},
+			emptyObj:    &Int3Struct{},
+			expectBytes: []byte{0, 0, 3},
+		},
+		{
+			name:        "ArrayOfCustomType",
+			packObj:     [2]Int3{3, 4},
+			emptyObj:    [2]Int3{},
+			expectBytes: []byte{0, 0, 3, 0, 0, 4},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "PointerToArrayOfCustomType",
+			packObj:     &[2]Int3{3, 4},
+			emptyObj:    &[2]Int3{},
+			expectBytes: []byte{0, 0, 3, 0, 0, 4},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "ArrayOfCustomTypeStruct",
+			packObj:     &ArrayInt3Struct{[2]Int3{3, 4}},
+			emptyObj:    &ArrayInt3Struct{},
+			expectBytes: []byte{0, 0, 3, 0, 0, 4},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "CustomTypeOfArrayOfUInt8",
+			packObj:     &DoubleUInt8{32, 64},
+			emptyObj:    &DoubleUInt8{},
+			expectBytes: []byte{32, 64},
+		},
+		{
+			name:        "CustomTypeOfArrayOfUInt8Struct",
+			packObj:     &DoubleUInt8Struct{I: DoubleUInt8{32, 64}},
+			emptyObj:    &DoubleUInt8Struct{},
+			expectBytes: []byte{32, 64},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "CustomTypeOfArrayOfCustomType",
+			packObj:     &DoubleInt3{Int3(128), Int3(256)},
+			emptyObj:    &DoubleInt3{},
+			expectBytes: []byte{0, 0, 128, 0, 1, 0},
+		},
+		{
+			name:        "CustomTypeOfArrayOfCustomTypeStruct",
+			packObj:     &DoubleInt3Struct{D: DoubleInt3{Int3(128), Int3(256)}},
+			emptyObj:    &DoubleInt3Struct{},
+			expectBytes: []byte{0, 0, 128, 0, 1, 0},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "CustomTypeOfSliceOfUInt8",
+			packObj:     &SliceUInt8{128, 64, 32},
+			emptyObj:    &SliceUInt8{},
+			expectBytes: []byte{128, 64, 32, 0},
+		},
+		{
+			name:        "CustomTypeOfSliceOfUInt8-Empty",
+			packObj:     &SliceUInt8{},
+			emptyObj:    &SliceUInt8{},
+			expectBytes: []byte{0},
+		},
+		{
+			name:        "CustomTypeOfSliceOfUInt8Struct",
+			packObj:     &SliceUInt8Struct{I: SliceUInt8{128, 64, 32}, N: 192},
+			emptyObj:    &SliceUInt8Struct{},
+			expectBytes: []byte{128, 64, 32, 0, 192},
+			skip:        true, // Not implemented.
+		},
+		{
+			name:        "CustomTypeOfChar4Struct",
+			packObj:     &Char4Struct{C: Char4("foo\x00")},
+			emptyObj:    &Char4Struct{},
+			expectBytes: []byte{102, 111, 111, 0},
+		},
+	}
+
+	for _, test := range testCases {
+		// TODO: Switch to t.Run() when Go 1.7 is the minimum supported version.
+		t.Log("RUN ", test.name)
+		runner := func(t *testing.T) {
+			defer func() {
+				if r := recover(); r != nil {
+					t.Log("unexpected panic:", r)
+					t.Error(r)
+				}
+			}()
+			if test.skip {
+				// TODO: Switch to t.Skip() when Go 1.7 is supported
+				t.Log("skipped unimplemented")
+				return
+			}
+			var buf bytes.Buffer
+			if err := Pack(&buf, test.packObj); err != nil {
+				t.Fatal(err)
+			}
+			if !bytes.Equal(buf.Bytes(), test.expectBytes) {
+				t.Fatal("error packing, expect:", test.expectBytes, "found:", buf.Bytes())
+			}
+			if err := Unpack(&buf, test.emptyObj); err != nil {
+				t.Fatal(err)
+			}
+			if !reflect.DeepEqual(test.packObj, test.emptyObj) {
+				t.Fatal("error unpacking, expect:", test.packObj, "found:", test.emptyObj)
+			}
+		}
+		runner(t)
+	}
+}

common/utils/sturc/field.go

@@ -0,0 +1,288 @@
+package struc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"math"
+	"reflect"
+)
+
+type Field struct {
+	Name     string
+	Ptr      bool
+	Index    int
+	Type     Type
+	defType  Type
+	Array    bool
+	Slice    bool
+	Len      int
+	Order    binary.ByteOrder
+	Sizeof   []int
+	Sizefrom []int
+	Fields   Fields
+	kind     reflect.Kind
+}
+
+func (f *Field) String() string {
+	var out string
+	if f.Type == Pad {
+		return fmt.Sprintf("{type: Pad, len: %d}", f.Len)
+	} else {
+		out = fmt.Sprintf("type: %s, order: %v", f.Type.String(), f.Order)
+	}
+	if f.Sizefrom != nil {
+		out += fmt.Sprintf(", sizefrom: %v", f.Sizefrom)
+	} else if f.Len > 0 {
+		out += fmt.Sprintf(", len: %d", f.Len)
+	}
+	if f.Sizeof != nil {
+		out += fmt.Sprintf(", sizeof: %v", f.Sizeof)
+	}
+	return "{" + out + "}"
+}
+
+func (f *Field) Size(val reflect.Value, options *Options) int {
+	typ := f.Type.Resolve(options)
+	size := 0
+	if typ == Struct {
+		vals := []reflect.Value{val}
+		if f.Slice {
+			vals = make([]reflect.Value, val.Len())
+			for i := 0; i < val.Len(); i++ {
+				vals[i] = val.Index(i)
+			}
+		}
+		for _, val := range vals {
+			size += f.Fields.Sizeof(val, options)
+		}
+	} else if typ == Pad {
+		size = f.Len
+	} else if typ == CustomType {
+		return val.Addr().Interface().(Custom).Size(options)
+	} else if f.Slice || f.kind == reflect.String {
+		length := val.Len()
+		if f.Len > 1 {
+			length = f.Len
+		}
+		size = length * typ.Size()
+	} else {
+		size = typ.Size()
+	}
+	align := options.ByteAlign
+	if align > 0 && size < align {
+		size = align
+	}
+	return size
+}
+
+func (f *Field) packVal(buf []byte, val reflect.Value, length int, options *Options) (size int, err error) {
+	order := f.Order
+	if options.Order != nil {
+		order = options.Order
+	}
+	if f.Ptr {
+		val = val.Elem()
+	}
+	typ := f.Type.Resolve(options)
+	switch typ {
+	case Struct:
+		return f.Fields.Pack(buf, val, options)
+	case Bool, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64:
+		size = typ.Size()
+		var n uint64
+		switch f.kind {
+		case reflect.Bool:
+			if val.Bool() {
+				n = 1
+			} else {
+				n = 0
+			}
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			n = uint64(val.Int())
+		default:
+			n = val.Uint()
+		}
+		switch typ {
+		case Bool:
+			if n != 0 {
+				buf[0] = 1
+			} else {
+				buf[0] = 0
+			}
+		case Int8, Uint8:
+			buf[0] = byte(n)
+		case Int16, Uint16:
+			order.PutUint16(buf, uint16(n))
+		case Int32, Uint32:
+			order.PutUint32(buf, uint32(n))
+		case Int64, Uint64:
+			order.PutUint64(buf, uint64(n))
+		}
+	case Float32, Float64:
+		size = typ.Size()
+		n := val.Float()
+		switch typ {
+		case Float32:
+			order.PutUint32(buf, math.Float32bits(float32(n)))
+		case Float64:
+			order.PutUint64(buf, math.Float64bits(n))
+		}
+	case String:
+		switch f.kind {
+		case reflect.String:
+			size = val.Len()
+			copy(buf, []byte(val.String()))
+		default:
+			// TODO: handle kind != bytes here
+			size = val.Len()
+			copy(buf, val.Bytes())
+		}
+	case CustomType:
+		return val.Addr().Interface().(Custom).Pack(buf, options)
+	default:
+		panic(fmt.Sprintf("no pack handler for type: %s", typ))
+	}
+	return
+}
+
+func (f *Field) Pack(buf []byte, val reflect.Value, length int, options *Options) (int, error) {
+	typ := f.Type.Resolve(options)
+	if typ == Pad {
+		for i := 0; i < length; i++ {
+			buf[i] = 0
+		}
+		return length, nil
+	}
+	if f.Slice {
+		// special case strings and byte slices for performance
+		end := val.Len()
+		if !f.Array && typ == Uint8 && (f.defType == Uint8 || f.kind == reflect.String) {
+			var tmp []byte
+			if f.kind == reflect.String {
+				tmp = []byte(val.String())
+			} else {
+				tmp = val.Bytes()
+			}
+			copy(buf, tmp)
+			if end < length {
+				// TODO: allow configuring pad byte?
+				rep := bytes.Repeat([]byte{0}, length-end)
+				copy(buf[end:], rep)
+				return length, nil
+			}
+			return val.Len(), nil
+		}
+		pos := 0
+		var zero reflect.Value
+		if end < length {
+			zero = reflect.Zero(val.Type().Elem())
+		}
+		for i := 0; i < length; i++ {
+			cur := zero
+			if i < end {
+				cur = val.Index(i)
+			}
+			if n, err := f.packVal(buf[pos:], cur, 1, options); err != nil {
+				return pos, err
+			} else {
+				pos += n
+			}
+		}
+		return pos, nil
+	} else {
+		return f.packVal(buf, val, length, options)
+	}
+}
+
+func (f *Field) unpackVal(buf []byte, val reflect.Value, length int, options *Options) error {
+	order := f.Order
+	if options.Order != nil {
+		order = options.Order
+	}
+	if f.Ptr {
+		val = val.Elem()
+	}
+	typ := f.Type.Resolve(options)
+	switch typ {
+	case Float32, Float64:
+		var n float64
+		switch typ {
+		case Float32:
+			n = float64(math.Float32frombits(order.Uint32(buf)))
+		case Float64:
+			n = math.Float64frombits(order.Uint64(buf))
+		}
+		switch f.kind {
+		case reflect.Float32, reflect.Float64:
+			val.SetFloat(n)
+		default:
+			return fmt.Errorf("struc: refusing to unpack float into field %s of type %s", f.Name, f.kind.String())
+		}
+	case Bool, Int8, Int16, Int32, Int64, Uint8, Uint16, Uint32, Uint64:
+		var n uint64
+		switch typ {
+		case Int8:
+			n = uint64(int64(int8(buf[0])))
+		case Int16:
+			n = uint64(int64(int16(order.Uint16(buf))))
+		case Int32:
+			n = uint64(int64(int32(order.Uint32(buf))))
+		case Int64:
+			n = uint64(int64(order.Uint64(buf)))
+		case Bool, Uint8:
+			n = uint64(buf[0])
+		case Uint16:
+			n = uint64(order.Uint16(buf))
+		case Uint32:
+			n = uint64(order.Uint32(buf))
+		case Uint64:
+			n = uint64(order.Uint64(buf))
+		}
+		switch f.kind {
+		case reflect.Bool:
+			val.SetBool(n != 0)
+		case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+			val.SetInt(int64(n))
+		default:
+			val.SetUint(n)
+		}
+	default:
+		panic(fmt.Sprintf("no unpack handler for type: %s", typ))
+	}
+	return nil
+}
+
+func (f *Field) Unpack(buf []byte, val reflect.Value, length int, options *Options) error {
+	typ := f.Type.Resolve(options)
+	if typ == Pad || f.kind == reflect.String {
+		if typ == Pad {
+			return nil
+		} else {
+			val.SetString(string(buf))
+			return nil
+		}
+	} else if f.Slice {
+		if val.Cap() < length {
+			val.Set(reflect.MakeSlice(val.Type(), length, length))
+		} else if val.Len() < length {
+			val.Set(val.Slice(0, length))
+		}
+		// special case byte slices for performance
+		if !f.Array && typ == Uint8 && f.defType == Uint8 {
+			copy(val.Bytes(), buf[:length])
+			return nil
+		}
+		pos := 0
+		size := typ.Size()
+		for i := 0; i < length; i++ {
+			if err := f.unpackVal(buf[pos:pos+size], val.Index(i), 1, options); err != nil {
+				return err
+			}
+			pos += size
+		}
+		return nil
+	} else {
+		return f.unpackVal(buf, val, length, options)
+	}
+}

common/utils/sturc/field_test.go

@@ -0,0 +1,77 @@
+package struc
+
+import (
+	"bytes"
+	"testing"
+)
+
+type badFloat struct {
+	BadFloat int `struc:"float64"`
+}
+
+func TestBadFloatField(t *testing.T) {
+	buf := bytes.NewReader([]byte("00000000"))
+	err := Unpack(buf, &badFloat{})
+	if err == nil {
+		t.Fatal("failed to error on bad float unpack")
+	}
+}
+
+type emptyLengthField struct {
+	Strlen int `struc:"sizeof=Str"`
+	Str    []byte
+}
+
+func TestEmptyLengthField(t *testing.T) {
+	var buf bytes.Buffer
+	s := &emptyLengthField{0, []byte("test")}
+	o := &emptyLengthField{}
+	if err := Pack(&buf, s); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, o); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(s.Str, o.Str) {
+		t.Fatal("empty length field encode failed")
+	}
+}
+
+type fixedSlicePad struct {
+	Field []byte `struc:"[4]byte"`
+}
+
+func TestFixedSlicePad(t *testing.T) {
+	var buf bytes.Buffer
+	ref := []byte{0, 0, 0, 0}
+	s := &fixedSlicePad{}
+	if err := Pack(&buf, s); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(buf.Bytes(), ref) {
+		t.Fatal("implicit fixed slice pack failed")
+	}
+	if err := Unpack(&buf, s); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(s.Field, ref) {
+		t.Fatal("implicit fixed slice unpack failed")
+	}
+}
+
+type sliceCap struct {
+	Len   int `struc:"sizeof=Field"`
+	Field []byte
+}
+
+func TestSliceCap(t *testing.T) {
+	var buf bytes.Buffer
+	tmp := &sliceCap{0, []byte("1234")}
+	if err := Pack(&buf, tmp); err != nil {
+		t.Fatal(err)
+	}
+	tmp.Field = make([]byte, 0, 4)
+	if err := Unpack(&buf, tmp); err != nil {
+		t.Fatal(err)
+	}
+}

common/utils/sturc/fields.go

@@ -0,0 +1,178 @@
+package struc
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"reflect"
+	"strings"
+)
+
+type Fields []*Field
+
+func (f Fields) SetByteOrder(order binary.ByteOrder) {
+	for _, field := range f {
+		if field != nil {
+			field.Order = order
+		}
+	}
+}
+
+func (f Fields) String() string {
+	fields := make([]string, len(f))
+	for i, field := range f {
+		if field != nil {
+			fields[i] = field.String()
+		}
+	}
+	return "{" + strings.Join(fields, ", ") + "}"
+}
+
+func (f Fields) Sizeof(val reflect.Value, options *Options) int {
+	for val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+	size := 0
+	for i, field := range f {
+		if field != nil {
+			size += field.Size(val.Field(i), options)
+		}
+	}
+	return size
+}
+
+func (f Fields) sizefrom(val reflect.Value, index []int) int {
+	field := val.FieldByIndex(index)
+	switch field.Kind() {
+	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+		return int(field.Int())
+	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+		n := int(field.Uint())
+		// all the builtin array length types are native int
+		// so this guards against weird truncation
+		if n < 0 {
+			return 0
+		}
+		return n
+	default:
+		name := val.Type().FieldByIndex(index).Name
+		panic(fmt.Sprintf("sizeof field %T.%s not an integer type", val.Interface(), name))
+	}
+}
+
+func (f Fields) Pack(buf []byte, val reflect.Value, options *Options) (int, error) {
+	for val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+	pos := 0
+	for i, field := range f {
+		if field == nil {
+			continue
+		}
+		v := val.Field(i)
+		length := field.Len
+		if field.Sizefrom != nil {
+			length = f.sizefrom(val, field.Sizefrom)
+		}
+		if length <= 0 && field.Slice {
+			length = v.Len()
+		}
+		if field.Sizeof != nil {
+			length := val.FieldByIndex(field.Sizeof).Len()
+			switch field.kind {
+			case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
+				// allocating a new int here has fewer side effects (doesn't update the original struct)
+				// but it's a wasteful allocation
+				// the old method might work if we just cast the temporary int/uint to the target type
+				v = reflect.New(v.Type()).Elem()
+				v.SetInt(int64(length))
+			case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
+				v = reflect.New(v.Type()).Elem()
+				v.SetUint(uint64(length))
+			default:
+				panic(fmt.Sprintf("sizeof field is not int or uint type: %s, %s", field.Name, v.Type()))
+			}
+		}
+		if n, err := field.Pack(buf[pos:], v, length, options); err != nil {
+			return n, err
+		} else {
+			pos += n
+			//g.Dump(v)
+		}
+	}
+	return pos, nil
+}
+
+func (f Fields) Unpack(r io.Reader, val reflect.Value, options *Options) error {
+	for val.Kind() == reflect.Ptr {
+		val = val.Elem()
+	}
+	var tmp [8]byte
+	var buf []byte
+	for i, field := range f {
+		if field == nil {
+			continue
+		}
+		v := val.Field(i)
+		length := field.Len
+		if field.Sizefrom != nil {
+			length = f.sizefrom(val, field.Sizefrom)
+		}
+		if v.Kind() == reflect.Ptr && !v.Elem().IsValid() {
+			v.Set(reflect.New(v.Type().Elem()))
+		}
+		if field.Type == Struct {
+			if field.Slice {
+				vals := v
+				if !field.Array {
+					vals = reflect.MakeSlice(v.Type(), length, length)
+				}
+				for i := 0; i < length; i++ {
+					v := vals.Index(i)
+					fields, err := parseFields(v)
+					if err != nil {
+						return err
+					}
+					if err := fields.Unpack(r, v, options); err != nil {
+						return err
+					}
+				}
+				if !field.Array {
+					v.Set(vals)
+				}
+			} else {
+				// TODO: DRY (we repeat the inner loop above)
+				fields, err := parseFields(v)
+				if err != nil {
+					return err
+				}
+				if err := fields.Unpack(r, v, options); err != nil {
+					return err
+				}
+			}
+			continue
+		} else {
+			typ := field.Type.Resolve(options)
+			if typ == CustomType {
+				if err := v.Addr().Interface().(Custom).Unpack(r, length, options); err != nil {
+					return err
+				}
+			} else {
+				size := length * field.Type.Resolve(options).Size()
+				if size < 8 {
+					buf = tmp[:size]
+				} else {
+					buf = make([]byte, size)
+				}
+				if _, err := io.ReadFull(r, buf); err != nil {
+					return err
+				}
+				err := field.Unpack(buf[:size], v, length, options)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}

common/utils/sturc/fields_test.go

@@ -0,0 +1,81 @@
+package struc
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+var refVal = reflect.ValueOf(reference)
+
+func TestFieldsParse(t *testing.T) {
+	if _, err := parseFields(refVal); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func TestFieldsString(t *testing.T) {
+	fields, _ := parseFields(refVal)
+	fields.String()
+}
+
+type sizefromStruct struct {
+	Size1 uint `struc:"sizeof=Var1"`
+	Var1  []byte
+	Size2 int `struc:"sizeof=Var2"`
+	Var2  []byte
+}
+
+func TestFieldsSizefrom(t *testing.T) {
+	var test = sizefromStruct{
+		Var1: []byte{1, 2, 3},
+		Var2: []byte{4, 5, 6},
+	}
+	var buf bytes.Buffer
+	err := Pack(&buf, &test)
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = Unpack(&buf, &test)
+	if err != nil {
+		t.Fatal(err)
+	}
+}
+
+type sizefromStructBad struct {
+	Size1 string `struc:"sizeof=Var1"`
+	Var1  []byte
+}
+
+func TestFieldsSizefromBad(t *testing.T) {
+	var test = &sizefromStructBad{Var1: []byte{1, 2, 3}}
+	var buf bytes.Buffer
+	defer func() {
+		if err := recover(); err == nil {
+			t.Fatal("failed to panic on bad sizeof type")
+		}
+	}()
+	Pack(&buf, &test)
+}
+
+type StructWithinArray struct {
+	a uint32
+}
+
+type StructHavingArray struct {
+	Props [1]StructWithinArray `struc:"[1]StructWithinArray"`
+}
+
+func TestStrucArray(t *testing.T) {
+	var buf bytes.Buffer
+	a := &StructHavingArray{[1]StructWithinArray{}}
+	err := Pack(&buf, a)
+	if err != nil {
+		t.Fatal(err)
+	}
+	b := &StructHavingArray{}
+	err = Unpack(&buf, b)
+	if err != nil {
+		t.Fatal(err)
+	}
+}

common/utils/sturc/go.mod

@@ -0,0 +1,3 @@
+module github.com/lunixbochs/struc
+
+go 1.12

common/utils/sturc/legacy.go

@@ -0,0 +1,16 @@
+package struc
+
+import (
+	"encoding/binary"
+	"io"
+)
+
+// Deprecated. Use PackWithOptions.
+func PackWithOrder(w io.Writer, data interface{}, order binary.ByteOrder) error {
+	return PackWithOptions(w, data, &Options{Order: order})
+}
+
+// Deprecated. Use UnpackWithOptions.
+func UnpackWithOrder(r io.Reader, data interface{}, order binary.ByteOrder) error {
+	return UnpackWithOptions(r, data, &Options{Order: order})
+}

common/utils/sturc/packable_test.go

@@ -0,0 +1,123 @@
+package struc
+
+import (
+	"bytes"
+	"fmt"
+	"testing"
+)
+
+var packableReference = []byte{
+	1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 5, 0, 6, 0, 0, 0, 7, 0, 0, 0, 0, 0, 0, 0, 8,
+	9, 10, 11, 12, 13, 14, 15, 16,
+	0, 17, 0, 18, 0, 19, 0, 20, 0, 21, 0, 22, 0, 23, 0, 24,
+}
+
+func TestPackable(t *testing.T) {
+	var (
+		buf bytes.Buffer
+
+		i8  int8   = 1
+		i16 int16  = 2
+		i32 int32  = 3
+		i64 int64  = 4
+		u8  uint8  = 5
+		u16 uint16 = 6
+		u32 uint32 = 7
+		u64 uint64 = 8
+
+		u8a  = [8]uint8{9, 10, 11, 12, 13, 14, 15, 16}
+		u16a = [8]uint16{17, 18, 19, 20, 21, 22, 23, 24}
+	)
+	// pack tests
+	if err := Pack(&buf, i8); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, i16); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, i32); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, i64); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, u8); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, u16); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, u32); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, u64); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, u8a[:]); err != nil {
+		t.Fatal(err)
+	}
+	if err := Pack(&buf, u16a[:]); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(buf.Bytes(), packableReference) {
+		fmt.Println(buf.Bytes())
+		fmt.Println(packableReference)
+		t.Fatal("Packable Pack() did not match reference.")
+	}
+	// unpack tests
+	i8 = 0
+	i16 = 0
+	i32 = 0
+	i64 = 0
+	u8 = 0
+	u16 = 0
+	u32 = 0
+	u64 = 0
+	if err := Unpack(&buf, &i8); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &i16); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &i32); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &i64); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &u8); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &u16); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &u32); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, &u64); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, u8a[:]); err != nil {
+		t.Fatal(err)
+	}
+	if err := Unpack(&buf, u16a[:]); err != nil {
+		t.Fatal(err)
+	}
+	// unpack checks
+	if i8 != 1 || i16 != 2 || i32 != 3 || i64 != 4 {
+		t.Fatal("Signed integer unpack failed.")
+	}
+	if u8 != 5 || u16 != 6 || u32 != 7 || u64 != 8 {
+		t.Fatal("Unsigned integer unpack failed.")
+	}
+	for i := 0; i < 8; i++ {
+		if u8a[i] != uint8(i+9) {
+			t.Fatal("uint8 array unpack failed.")
+		}
+	}
+	for i := 0; i < 8; i++ {
+		if u16a[i] != uint16(i+17) {
+			t.Fatal("uint16 array unpack failed.")
+		}
+	}
+}

common/utils/sturc/packer.go

@@ -0,0 +1,13 @@
+package struc
+
+import (
+	"io"
+	"reflect"
+)
+
+type Packer interface {
+	Pack(buf []byte, val reflect.Value, options *Options) (int, error)
+	Unpack(r io.Reader, val reflect.Value, options *Options) error
+	Sizeof(val reflect.Value, options *Options) int
+	String() string
+}

common/utils/sturc/parse.go

@@ -0,0 +1,230 @@
+package struc
+
+import (
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"reflect"
+	"regexp"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// struc:"int32,big,sizeof=Data,skip,sizefrom=Len"
+
+type strucTag struct {
+	Type     string
+	Order    binary.ByteOrder
+	Sizeof   string
+	Skip     bool
+	Sizefrom string
+}
+
+func parseStrucTag(tag reflect.StructTag) *strucTag {
+	t := &strucTag{
+		Order: binary.BigEndian,
+	}
+	tagStr := tag.Get("struc")
+	if tagStr == "" {
+		// someone's going to typo this (I already did once)
+		// sorry if you made a module actually using this tag
+		// and you're mad at me now
+		tagStr = tag.Get("struct")
+	}
+	for _, s := range strings.Split(tagStr, ",") {
+		if strings.HasPrefix(s, "sizeof=") {
+			tmp := strings.SplitN(s, "=", 2)
+			t.Sizeof = tmp[1]
+		} else if strings.HasPrefix(s, "sizefrom=") {
+			tmp := strings.SplitN(s, "=", 2)
+			t.Sizefrom = tmp[1]
+		} else if s == "big" {
+			t.Order = binary.BigEndian
+		} else if s == "little" {
+			t.Order = binary.LittleEndian
+		} else if s == "skip" {
+			t.Skip = true
+		} else {
+			t.Type = s
+		}
+	}
+	return t
+}
+
+var typeLenRe = regexp.MustCompile(`^\[(\d*)\]`)
+
+func parseField(f reflect.StructField) (fd *Field, tag *strucTag, err error) {
+	tag = parseStrucTag(f.Tag)
+	var ok bool
+	fd = &Field{
+		Name:  f.Name,
+		Len:   1,
+		Order: tag.Order,
+		Slice: false,
+		kind:  f.Type.Kind(),
+	}
+	switch fd.kind {
+	case reflect.Array:
+		fd.Slice = true
+		fd.Array = true
+		fd.Len = f.Type.Len()
+		fd.kind = f.Type.Elem().Kind()
+	case reflect.Slice:
+		fd.Slice = true
+		fd.Len = -1
+		fd.kind = f.Type.Elem().Kind()
+	case reflect.Ptr:
+		fd.Ptr = true
+		fd.kind = f.Type.Elem().Kind()
+	}
+	// check for custom types
+	tmp := reflect.New(f.Type)
+	if _, ok := tmp.Interface().(Custom); ok {
+		fd.Type = CustomType
+		return
+	}
+	var defTypeOk bool
+	fd.defType, defTypeOk = reflectTypeMap[fd.kind]
+	// find a type in the struct tag
+	pureType := typeLenRe.ReplaceAllLiteralString(tag.Type, "")
+	if fd.Type, ok = typeLookup[pureType]; ok {
+		fd.Len = 1
+		match := typeLenRe.FindAllStringSubmatch(tag.Type, -1)
+		if len(match) > 0 && len(match[0]) > 1 {
+			fd.Slice = true
+			first := match[0][1]
+			// Field.Len = -1 indicates a []slice
+			if first == "" {
+				fd.Len = -1
+			} else {
+				fd.Len, err = strconv.Atoi(first)
+			}
+		}
+		return
+	}
+	// the user didn't specify a type
+	switch f.Type {
+	case reflect.TypeOf(Size_t(0)):
+		fd.Type = SizeType
+	case reflect.TypeOf(Off_t(0)):
+		fd.Type = OffType
+	default:
+		if defTypeOk {
+			fd.Type = fd.defType
+		} else {
+			err = errors.New(fmt.Sprintf("struc: Could not resolve field '%v' type '%v'.", f.Name, f.Type))
+		}
+	}
+	return
+}
+
+func parseFieldsLocked(v reflect.Value) (Fields, error) {
+	// we need to repeat this logic because parseFields() below can't be recursively called due to locking
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	t := v.Type()
+	if v.NumField() < 1 {
+		return nil, errors.New("struc: Struct has no fields.")
+	}
+	sizeofMap := make(map[string][]int)
+	fields := make(Fields, v.NumField())
+	for i := 0; i < t.NumField(); i++ {
+		field := t.Field(i)
+		f, tag, err := parseField(field)
+		if tag.Skip {
+			continue
+		}
+		if err != nil {
+			return nil, err
+		}
+		if !v.Field(i).CanSet() {
+			continue
+		}
+		f.Index = i
+		if tag.Sizeof != "" {
+			target, ok := t.FieldByName(tag.Sizeof)
+			if !ok {
+				return nil, fmt.Errorf("struc: `sizeof=%s` field does not exist", tag.Sizeof)
+			}
+			f.Sizeof = target.Index
+			sizeofMap[tag.Sizeof] = field.Index
+		}
+		if sizefrom, ok := sizeofMap[field.Name]; ok {
+			f.Sizefrom = sizefrom
+		}
+		if tag.Sizefrom != "" {
+			source, ok := t.FieldByName(tag.Sizefrom)
+			if !ok {
+				return nil, fmt.Errorf("struc: `sizefrom=%s` field does not exist", tag.Sizefrom)
+			}
+			f.Sizefrom = source.Index
+		}
+		if f.Len == -1 && f.Sizefrom == nil {
+			return nil, fmt.Errorf("struc: field `%s` is a slice with no length or sizeof field", field.Name)
+		}
+		// recurse into nested structs
+		// TODO: handle loops (probably by indirecting the []Field and putting pointer in cache)
+		if f.Type == Struct {
+			typ := field.Type
+			if f.Ptr {
+				typ = typ.Elem()
+			}
+			if f.Slice {
+				typ = typ.Elem()
+			}
+			f.Fields, err = parseFieldsLocked(reflect.New(typ))
+			if err != nil {
+				return nil, err
+			}
+		}
+		fields[i] = f
+	}
+	return fields, nil
+}
+
+var fieldCache = make(map[reflect.Type]Fields)
+var fieldCacheLock sync.RWMutex
+var parseLock sync.Mutex
+
+func fieldCacheLookup(t reflect.Type) Fields {
+	fieldCacheLock.RLock()
+	defer fieldCacheLock.RUnlock()
+	if cached, ok := fieldCache[t]; ok {
+		return cached
+	}
+	return nil
+}
+
+func parseFields(v reflect.Value) (Fields, error) {
+	for v.Kind() == reflect.Ptr {
+		v = v.Elem()
+	}
+	t := v.Type()
+
+	// fast path: hopefully the field parsing is already cached
+	if cached := fieldCacheLookup(t); cached != nil {
+		return cached, nil
+	}
+
+	// hold a global lock so multiple goroutines can't parse (the same) fields at once
+	parseLock.Lock()
+	defer parseLock.Unlock()
+
+	// check cache a second time, in case parseLock was just released by
+	// another thread who filled the cache for us
+	if cached := fieldCacheLookup(t); cached != nil {
+		return cached, nil
+	}
+
+	// no luck, time to parse and fill the cache ourselves
+	fields, err := parseFieldsLocked(v)
+	if err != nil {
+		return nil, err
+	}
+	fieldCacheLock.Lock()
+	fieldCache[t] = fields
+	fieldCacheLock.Unlock()
+	return fields, nil
+}

common/utils/sturc/parse_test.go

@@ -0,0 +1,62 @@
+package struc
+
+import (
+	"bytes"
+	"reflect"
+	"testing"
+)
+
+func parseTest(data interface{}) error {
+	_, err := parseFields(reflect.ValueOf(data))
+	return err
+}
+
+type empty struct{}
+
+func TestEmptyStruc(t *testing.T) {
+	if err := parseTest(&empty{}); err == nil {
+		t.Fatal("failed to error on empty struct")
+	}
+}
+
+type chanStruct struct {
+	Test chan int
+}
+
+func TestChanError(t *testing.T) {
+	if err := parseTest(&chanStruct{}); err == nil {
+		// TODO: should probably ignore channel fields
+		t.Fatal("failed to error on struct containing channel")
+	}
+}
+
+type badSizeof struct {
+	Size int `struc:"sizeof=Bad"`
+}
+
+func TestBadSizeof(t *testing.T) {
+	if err := parseTest(&badSizeof{}); err == nil {
+		t.Fatal("failed to error on missing Sizeof target")
+	}
+}
+
+type missingSize struct {
+	Test []byte
+}
+
+func TestMissingSize(t *testing.T) {
+	if err := parseTest(&missingSize{}); err == nil {
+		t.Fatal("failed to error on missing field size")
+	}
+}
+
+type badNested struct {
+	Empty empty
+}
+
+func TestNestedParseError(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Pack(&buf, &badNested{}); err == nil {
+		t.Fatal("failed to error on bad nested struct")
+	}
+}

common/utils/sturc/struc.go

@@ -0,0 +1,122 @@
+package struc
+
+import (
+	"encoding/binary"
+	"fmt"
+	"io"
+	"reflect"
+)
+
+type Options struct {
+	ByteAlign int
+	PtrSize   int
+	Order     binary.ByteOrder
+}
+
+func (o *Options) Validate() error {
+	if o.PtrSize == 0 {
+		o.PtrSize = 32
+	} else {
+		switch o.PtrSize {
+		case 8, 16, 32, 64:
+		default:
+			return fmt.Errorf("Invalid Options.PtrSize: %d. Must be in (8, 16, 32, 64)", o.PtrSize)
+		}
+	}
+	return nil
+}
+
+var emptyOptions = &Options{}
+
+func init() {
+	// fill default values to avoid data race to be reported by race detector.
+	emptyOptions.Validate()
+}
+
+func prep(data interface{}) (reflect.Value, Packer, error) {
+	value := reflect.ValueOf(data)
+	for value.Kind() == reflect.Ptr {
+		next := value.Elem().Kind()
+		if next == reflect.Struct || next == reflect.Ptr {
+			value = value.Elem()
+		} else {
+			break
+		}
+	}
+	switch value.Kind() {
+	case reflect.Struct:
+		fields, err := parseFields(value)
+		return value, fields, err
+	default:
+		if !value.IsValid() {
+			return reflect.Value{}, nil, fmt.Errorf("Invalid reflect.Value for %+v", data)
+		}
+		if c, ok := data.(Custom); ok {
+			return value, customFallback{c}, nil
+		}
+		return value, binaryFallback(value), nil
+	}
+}
+
+func Pack(w io.Writer, data interface{}) error {
+	return PackWithOptions(w, data, nil)
+}
+
+func PackWithOptions(w io.Writer, data interface{}, options *Options) error {
+	if options == nil {
+		options = emptyOptions
+	}
+	if err := options.Validate(); err != nil {
+		return err
+	}
+	val, packer, err := prep(data)
+	if err != nil {
+		return err
+	}
+	if val.Type().Kind() == reflect.String {
+		val = val.Convert(reflect.TypeOf([]byte{}))
+	}
+	size := packer.Sizeof(val, options)
+	buf := make([]byte, size)
+	if _, err := packer.Pack(buf, val, options); err != nil {
+		return err
+	}
+	_, err = w.Write(buf)
+	return err
+}
+
+func Unpack(r io.Reader, data interface{}) error {
+	return UnpackWithOptions(r, data, nil)
+}
+
+func UnpackWithOptions(r io.Reader, data interface{}, options *Options) error {
+	if options == nil {
+		options = emptyOptions
+	}
+	if err := options.Validate(); err != nil {
+		return err
+	}
+	val, packer, err := prep(data)
+	if err != nil {
+		return err
+	}
+	return packer.Unpack(r, val, options)
+}
+
+func Sizeof(data interface{}) (int, error) {
+	return SizeofWithOptions(data, nil)
+}
+
+func SizeofWithOptions(data interface{}, options *Options) (int, error) {
+	if options == nil {
+		options = emptyOptions
+	}
+	if err := options.Validate(); err != nil {
+		return 0, err
+	}
+	val, packer, err := prep(data)
+	if err != nil {
+		return 0, err
+	}
+	return packer.Sizeof(val, options), nil
+}

common/utils/sturc/struc_test.go

@@ -0,0 +1,310 @@
+package struc
+
+import (
+	"bytes"
+	"encoding/binary"
+	"fmt"
+	"reflect"
+	"testing"
+)
+
+type Nested struct {
+	Test2 int `struc:"int8"`
+}
+
+type Example struct {
+	Pad    []byte `struc:"[5]pad"`        // 00 00 00 00 00
+	I8f    int    `struc:"int8"`          // 01
+	I16f   int    `struc:"int16"`         // 00 02
+	I32f   int    `struc:"int32"`         // 00 00 00 03
+	I64f   int    `struc:"int64"`         // 00 00 00 00 00 00 00 04
+	U8f    int    `struc:"uint8,little"`  // 05
+	U16f   int    `struc:"uint16,little"` // 06 00
+	U32f   int    `struc:"uint32,little"` // 07 00 00 00
+	U64f   int    `struc:"uint64,little"` // 08 00 00 00 00 00 00 00
+	Boolf  int    `struc:"bool"`          // 01
+	Byte4f []byte `struc:"[4]byte"`       // "abcd"
+
+	I8     int8    // 09
+	I16    int16   // 00 0a
+	I32    int32   // 00 00 00 0b
+	I64    int64   // 00 00 00 00 00 00 00 0c
+	U8     uint8   `struc:"little"` // 0d
+	U16    uint16  `struc:"little"` // 0e 00
+	U32    uint32  `struc:"little"` // 0f 00 00 00
+	U64    uint64  `struc:"little"` // 10 00 00 00 00 00 00 00
+	BoolT  bool    // 01
+	BoolF  bool    // 00
+	Byte4  [4]byte // "efgh"
+	Float1 float32 // 41 a0 00 00
+	Float2 float64 // 41 35 00 00 00 00 00 00
+
+	I32f2 int64 `struc:"int32"`  // ff ff ff ff
+	U32f2 int64 `struc:"uint32"` // ff ff ff ff
+
+	I32f3 int32 `struc:"int64"` // ff ff ff ff ff ff ff ff
+
+	Size int    `struc:"sizeof=Str,little"` // 0a 00 00 00
+	Str  string `struc:"[]byte"`            // "ijklmnopqr"
+	Strb string `struc:"[4]byte"`           // "stuv"
+
+	Size2 int    `struc:"uint8,sizeof=Str2"` // 04
+	Str2  string // "1234"
+
+	Size3 int    `struc:"uint8,sizeof=Bstr"` // 04
+	Bstr  []byte // "5678"
+
+	Size4 int    `struc:"little"`                // 07 00 00 00
+	Str4a string `struc:"[]byte,sizefrom=Size4"` // "ijklmno"
+	Str4b string `struc:"[]byte,sizefrom=Size4"` // "pqrstuv"
+
+	Size5 int    `struc:"uint8"`          // 04
+	Bstr2 []byte `struc:"sizefrom=Size5"` // "5678"
+
+	Nested  Nested  // 00 00 00 01
+	NestedP *Nested // 00 00 00 02
+	TestP64 *int    `struc:"int64"` // 00 00 00 05
+
+	NestedSize int      `struc:"sizeof=NestedA"` // 00 00 00 02
+	NestedA    []Nested // [00 00 00 03, 00 00 00 04]
+
+	Skip int `struc:"skip"`
+
+	CustomTypeSize    Int3   `struc:"sizeof=CustomTypeSizeArr"` // 00 00 00 04
+	CustomTypeSizeArr []byte // "ABCD"
+}
+
+var five = 5
+
+type ExampleStructWithin struct {
+	a uint8
+}
+
+type ExampleSlice struct {
+	PropsLen uint8 `struc:"sizeof=Props"`
+	Props    []ExampleStructWithin
+}
+
+type ExampleArray struct {
+	PropsLen uint8
+	Props    [16]ExampleStructWithin `struc:"[16]ExampleStructWithin"`
+}
+
+var arraySliceReferenceBytes = []byte{
+	16,
+	0, 0, 0, 1,
+	0, 0, 0, 1,
+	0, 0, 0, 2,
+	0, 0, 0, 3,
+	0, 0, 0, 4,
+	0, 0, 0, 5,
+	0, 0, 0, 6,
+	0, 0, 0, 7,
+	0, 0, 0, 8,
+	0, 0, 0, 9,
+	0, 0, 0, 10,
+	0, 0, 0, 11,
+	0, 0, 0, 12,
+	0, 0, 0, 13,
+	0, 0, 0, 14,
+	0, 0, 0, 15,
+	0, 0, 0, 16,
+}
+
+var arrayReference = &ExampleArray{
+	16,
+	[16]ExampleStructWithin{
+		ExampleStructWithin{1},
+		ExampleStructWithin{2},
+		ExampleStructWithin{3},
+		ExampleStructWithin{4},
+		ExampleStructWithin{5},
+		ExampleStructWithin{6},
+		ExampleStructWithin{7},
+		ExampleStructWithin{8},
+		ExampleStructWithin{9},
+		ExampleStructWithin{10},
+		ExampleStructWithin{11},
+		ExampleStructWithin{12},
+		ExampleStructWithin{13},
+		ExampleStructWithin{14},
+		ExampleStructWithin{15},
+		ExampleStructWithin{16},
+	},
+}
+
+var sliceReference = &ExampleSlice{
+	16,
+	[]ExampleStructWithin{
+		ExampleStructWithin{1},
+		ExampleStructWithin{2},
+		ExampleStructWithin{3},
+		ExampleStructWithin{4},
+		ExampleStructWithin{5},
+		ExampleStructWithin{6},
+		ExampleStructWithin{7},
+		ExampleStructWithin{8},
+		ExampleStructWithin{9},
+		ExampleStructWithin{10},
+		ExampleStructWithin{11},
+		ExampleStructWithin{12},
+		ExampleStructWithin{13},
+		ExampleStructWithin{14},
+		ExampleStructWithin{15},
+		ExampleStructWithin{16},
+	},
+}
+
+var reference = &Example{
+	nil,
+	1, 2, 3, 4, 5, 6, 7, 8, 0, []byte{'a', 'b', 'c', 'd'},
+	9, 10, 11, 12, 13, 14, 15, 16, true, false, [4]byte{'e', 'f', 'g', 'h'},
+	20, 21,
+	-1,
+	4294967295,
+	-1,
+	10, "ijklmnopqr", "stuv",
+	4, "1234",
+	4, []byte("5678"),
+	7, "ijklmno", "pqrstuv",
+	4, []byte("5678"),
+	Nested{1}, &Nested{2}, &five,
+	6, []Nested{{3}, {4}, {5}, {6}, {7}, {8}},
+	0,
+	Int3(4), []byte("ABCD"),
+}
+
+var referenceBytes = []byte{
+	0, 0, 0, 0, 0, // pad(5)
+	1, 0, 2, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, // fake int8-int64(1-4)
+	5, 6, 0, 7, 0, 0, 0, 8, 0, 0, 0, 0, 0, 0, 0, // fake little-endian uint8-uint64(5-8)
+	0,                  // fake bool(0)
+	'a', 'b', 'c', 'd', // fake [4]byte
+
+	9, 0, 10, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 12, // real int8-int64(9-12)
+	13, 14, 0, 15, 0, 0, 0, 16, 0, 0, 0, 0, 0, 0, 0, // real little-endian uint8-uint64(13-16)
+	1, 0, // real bool(1), bool(0)
+	'e', 'f', 'g', 'h', // real [4]byte
+	65, 160, 0, 0, // real float32(20)
+	64, 53, 0, 0, 0, 0, 0, 0, // real float64(21)
+
+	255, 255, 255, 255, // fake int32(-1)
+	255, 255, 255, 255, // fake uint32(4294967295)
+
+	255, 255, 255, 255, 255, 255, 255, 255, // fake int64(-1)
+
+	10, 0, 0, 0, // little-endian int32(10) sizeof=Str
+	'i', 'j', 'k', 'l', 'm', 'n', 'o', 'p', 'q', 'r', // Str
+	's', 't', 'u', 'v', // fake string([4]byte)
+	04, '1', '2', '3', '4', // real string
+	04, '5', '6', '7', '8', // fake []byte(string)
+
+	7, 0, 0, 0, // little-endian int32(7)
+	'i', 'j', 'k', 'l', 'm', 'n', 'o', // Str4a sizefrom=Size4
+	'p', 'q', 'r', 's', 't', 'u', 'v', // Str4b sizefrom=Size4
+	04, '5', '6', '7', '8', // fake []byte(string)
+
+	1, 2, // Nested{1}, Nested{2}
+	0, 0, 0, 0, 0, 0, 0, 5, // &five
+
+	0, 0, 0, 6, // int32(6)
+	3, 4, 5, 6, 7, 8, // [Nested{3}, ...Nested{8}]
+
+	0, 0, 4, 'A', 'B', 'C', 'D', // Int3(4), []byte("ABCD")
+}
+
+func TestCodec(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Pack(&buf, reference); err != nil {
+		t.Fatal(err)
+	}
+	out := &Example{}
+	if err := Unpack(&buf, out); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(reference, out) {
+		fmt.Printf("got: %#v\nwant: %#v\n", out, reference)
+		t.Fatal("encode/decode failed")
+	}
+}
+
+func TestEncode(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Pack(&buf, reference); err != nil {
+		t.Fatal(err)
+	}
+	if !bytes.Equal(buf.Bytes(), referenceBytes) {
+		fmt.Printf("got: %#v\nwant: %#v\n", buf.Bytes(), referenceBytes)
+		t.Fatal("encode failed")
+	}
+}
+
+func TestDecode(t *testing.T) {
+	buf := bytes.NewReader(referenceBytes)
+	out := &Example{}
+	if err := Unpack(buf, out); err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(reference, out) {
+		fmt.Printf("got: %#v\nwant: %#v\n", out, reference)
+		t.Fatal("decode failed")
+	}
+}
+
+func TestSizeof(t *testing.T) {
+	size, err := Sizeof(reference)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if size != len(referenceBytes) {
+		t.Fatalf("sizeof failed; expected %d, got %d", len(referenceBytes), size)
+	}
+}
+
+type ExampleEndian struct {
+	T int `struc:"int16,big"`
+}
+
+func TestEndianSwap(t *testing.T) {
+	var buf bytes.Buffer
+	big := &ExampleEndian{1}
+	if err := PackWithOrder(&buf, big, binary.BigEndian); err != nil {
+		t.Fatal(err)
+	}
+	little := &ExampleEndian{}
+	if err := UnpackWithOrder(&buf, little, binary.LittleEndian); err != nil {
+		t.Fatal(err)
+	}
+	if little.T != 256 {
+		t.Fatal("big -> little conversion failed")
+	}
+}
+
+func TestNilValue(t *testing.T) {
+	var buf bytes.Buffer
+	if err := Pack(&buf, nil); err == nil {
+		t.Fatal("failed throw error for bad struct value")
+	}
+	if err := Unpack(&buf, nil); err == nil {
+		t.Fatal("failed throw error for bad struct value")
+	}
+	if _, err := Sizeof(nil); err == nil {
+		t.Fatal("failed to throw error for bad struct value")
+	}
+}
+
+type sliceUnderrun struct {
+	Str string   `struc:"[10]byte"`
+	Arr []uint16 `struc:"[10]uint16"`
+}
+
+func TestSliceUnderrun(t *testing.T) {
+	var buf bytes.Buffer
+	v := sliceUnderrun{
+		Str: "foo",
+		Arr: []uint16{1, 2, 3},
+	}
+	if err := Pack(&buf, &v); err != nil {
+		t.Fatal(err)
+	}
+}

common/utils/sturc/test_pack_init/doc.go

@@ -0,0 +1,4 @@
+package test_pack_init
+
+// This is a placeholder package for a test on specific race detector report on
+// default Options initialization.

common/utils/sturc/test_pack_init/pack_init_test.go

@@ -0,0 +1,29 @@
+package test_pack_init
+
+import (
+	"bytes"
+	"github.com/lunixbochs/struc"
+	"sync"
+	"testing"
+)
+
+type Example struct {
+	I int `struc:int`
+}
+
+// TestParallelPack checks whether Pack is goroutine-safe. Run it with -race flag.
+// Keep it as a single test in package since it is likely to be triggered on initialization
+// of global objects reported as a data race by race detector.
+func TestParallelPack(t *testing.T) {
+	var wg sync.WaitGroup
+	val := Example{}
+	for i := 0; i < 2; i++ {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			var buf bytes.Buffer
+			_ = struc.Pack(&buf, &val)
+		}()
+	}
+	wg.Wait()
+}

common/utils/sturc/types.go

@@ -0,0 +1,136 @@
+package struc
+
+import (
+	"fmt"
+	"reflect"
+)
+
+type Type int
+
+const (
+	Invalid Type = iota
+	Pad
+	Bool
+	Int
+	Int8
+	Uint8
+	Int16
+	Uint16
+	Int32
+	Uint32
+	Int64
+	Uint64
+	Float32
+	Float64
+	String
+	Struct
+	Ptr
+
+	SizeType
+	OffType
+	CustomType
+)
+
+func (t Type) Resolve(options *Options) Type {
+	switch t {
+	case OffType:
+		switch options.PtrSize {
+		case 8:
+			return Int8
+		case 16:
+			return Int16
+		case 32:
+			return Int32
+		case 64:
+			return Int64
+		default:
+			panic(fmt.Sprintf("unsupported ptr bits: %d", options.PtrSize))
+		}
+	case SizeType:
+		switch options.PtrSize {
+		case 8:
+			return Uint8
+		case 16:
+			return Uint16
+		case 32:
+			return Uint32
+		case 64:
+			return Uint64
+		default:
+			panic(fmt.Sprintf("unsupported ptr bits: %d", options.PtrSize))
+		}
+	}
+	return t
+}
+
+func (t Type) String() string {
+	return typeNames[t]
+}
+
+func (t Type) Size() int {
+	switch t {
+	case SizeType, OffType:
+		panic("Size_t/Off_t types must be converted to another type using options.PtrSize")
+	case Pad, String, Int8, Uint8, Bool:
+		return 1
+	case Int16, Uint16:
+		return 2
+	case Int32, Uint32, Float32:
+		return 4
+	case Int64, Uint64, Float64:
+		return 8
+	default:
+		panic("Cannot resolve size of type:" + t.String())
+	}
+}
+
+var typeLookup = map[string]Type{
+	"pad":     Pad,
+	"bool":    Bool,
+	"byte":    Uint8,
+	"int8":    Int8,
+	"uint8":   Uint8,
+	"int16":   Int16,
+	"uint16":  Uint16,
+	"int32":   Int32,
+	"uint32":  Uint32,
+	"int64":   Int64,
+	"uint64":  Uint64,
+	"float32": Float32,
+	"float64": Float64,
+
+	"size_t": SizeType,
+	"off_t":  OffType,
+}
+
+var typeNames = map[Type]string{
+	CustomType: "Custom",
+}
+
+func init() {
+	for name, enum := range typeLookup {
+		typeNames[enum] = name
+	}
+}
+
+type Size_t uint64
+type Off_t int64
+
+var reflectTypeMap = map[reflect.Kind]Type{
+	reflect.Bool:    Bool,
+	reflect.Int8:    Int8,
+	reflect.Int16:   Int16,
+	reflect.Int:     Int32,
+	reflect.Int32:   Int32,
+	reflect.Int64:   Int64,
+	reflect.Uint8:   Uint8,
+	reflect.Uint16:  Uint16,
+	reflect.Uint:    Uint32,
+	reflect.Uint32:  Uint32,
+	reflect.Uint64:  Uint64,
+	reflect.Float32: Float32,
+	reflect.Float64: Float64,
+	reflect.String:  String,
+	reflect.Struct:  Struct,
+	reflect.Ptr:     Ptr,
+}

common/utils/sturc/types_test.go

@@ -0,0 +1,53 @@
+package struc
+
+import (
+	"bytes"
+	"testing"
+)
+
+func TestBadType(t *testing.T) {
+	defer func() { recover() }()
+	Type(-1).Size()
+	t.Fatal("failed to panic for invalid Type.Size()")
+}
+
+func TestTypeString(t *testing.T) {
+	if Pad.String() != "pad" {
+		t.Fatal("type string representation failed")
+	}
+}
+
+type sizeOffTest struct {
+	Size Size_t
+	Off  Off_t
+}
+
+func TestSizeOffTypes(t *testing.T) {
+	bits := []int{8, 16, 32, 64}
+	var buf bytes.Buffer
+	test := &sizeOffTest{1, 2}
+	for _, b := range bits {
+		if err := PackWithOptions(&buf, test, &Options{PtrSize: b}); err != nil {
+			t.Fatal(err)
+		}
+	}
+	reference := []byte{
+		1, 2,
+		0, 1, 0, 2,
+		0, 0, 0, 1, 0, 0, 0, 2,
+		0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 2,
+	}
+	if !bytes.Equal(reference, buf.Bytes()) {
+		t.Errorf("reference: %v != bytes: %v", reference, buf.Bytes())
+	}
+	reader := bytes.NewReader(buf.Bytes())
+	for _, b := range bits {
+		out := &sizeOffTest{}
+		if err := UnpackWithOptions(reader, out, &Options{PtrSize: b}); err != nil {
+			t.Fatal(err)
+		}
+		if out.Size != 1 || out.Off != 2 {
+			t.Errorf("Size_t/Off_t mismatch: {%d, %d}\n%v", out.Size, out.Off, buf.Bytes())
+		}
+	}
+}