//Unmarshals a buffer into a preallocated go friendly memory structure using the unsafe package.
func (this *m) UnmarshalInto(buf, mem []byte) error
This will be very unsafe and possibly go version dependant, but I think this will also be very fast.
This is some proof of concept work I did quite a while ago.
I just have not had the time to build this into a plugin.
I thought I'd better post it somewhere before I loose it.
package unmarshalinto
import (
"bytes"
"code.google.com/p/gogoprotobuf/proto"
"code.google.com/p/gogoprotobuf/test"
"math"
"reflect"
"testing"
"unsafe"
)
/*
type MyExtendable struct {
Field1 *int64 `protobuf:"varint,1,opt" json:"Field1,omitempty"`
XXX_extensions map[int32]proto.Extension `json:"-"`
XXX_unrecognized []byte `json:"-"`
}
extend MyExtendable {
optional double FieldA = 100;
optional NinOptNative FieldB = 101;
optional NinEmbeddedStruct FieldC = 102;
}
*/
var (
extMax = &test.MyExtendable{
Field1: proto.Int64(math.MaxInt64),
}
extB = &test.NinOptNative{
Field1: proto.Float64(math.MaxFloat64),
}
)
func TestUnmarshalIntoMyExtendable(t *testing.T) {
err := proto.SetExtension(extMax, test.E_FieldA, proto.Float64(math.MaxFloat64))
if err != nil {
panic(err)
}
err = proto.SetExtension(extMax, test.E_FieldB, extB)
if err != nil {
panic(err)
}
proto.GetRawExtension(extMax.XXX_extensions, 100)
proto.GetRawExtension(extMax.XXX_extensions, 101)
var msg *test.MyExtendable = nil
initSize := int(unsafe.Offsetof(msg.XXX_unrecognized) + unsafe.Sizeof(msg.XXX_unrecognized))
popSize := 1000
buf := make([]byte, initSize+popSize)
var offset int
msg = (*test.MyExtendable)(unsafe.Pointer(&buf[0]))
offset = initSize
msg.Field1 = (*int64)(unsafe.Pointer(&buf[offset]))
*msg.Field1 = math.MaxInt64
offset += int(unsafe.Sizeof(*msg.Field1))
//this is the key and wire type encoded as a varint
buf[offset] = 161
offset++
buf[offset] = 6
offset++
a := (*float64)(unsafe.Pointer(&buf[offset]))
offset += int(unsafe.Sizeof(*a))
*a = math.MaxFloat64
proto.SetRawExtension(msg, 100, buf[offset-10:offset])
t.Logf("%v", buf)
t.Logf("msg %#v", msg)
printWords(t, buf)
t.Logf("offset = %d offset/8 = %d", offset, offset/8)
if err := msg.VerboseEqual(extMax); err != nil {
panic(err)
}
}
/*
message NinOptStruct {
optional double Field1 = 1;
optional float Field2 = 2;
optional NidOptNative Field3 = 3;
optional NinOptNative Field4 = 4;
optional uint64 Field6 = 6;
optional sint32 Field7 = 7;
optional NidOptNative Field8 = 8;
optional bool Field13 = 13;
optional string Field14 = 14;
optional bytes Field15 = 15;
}
*/
var (
structMax2 = &test.NinOptStruct{
Field1: proto.Float64(math.MaxFloat64),
Field3: &test.NidOptNative{
Field1: math.MaxFloat64 / 2,
},
Field4: &test.NinOptNative{
Field2: proto.Float32(math.MaxFloat32 / 3),
},
Field6: proto.Uint64(math.MaxUint64 - 6),
}
)
func TestUnmarshalIntoNinOptStruct2(t *testing.T) {
var msg *test.NinOptStruct = nil
initSize := int(unsafe.Offsetof(msg.XXX_unrecognized) + unsafe.Sizeof(msg.XXX_unrecognized))
popSize := 1000
buf := make([]byte, initSize+popSize)
var offset int
msg = (*test.NinOptStruct)(unsafe.Pointer(&buf[0]))
offset = initSize
msg.Field1 = (*float64)(unsafe.Pointer(&buf[offset]))
*msg.Field1 = math.MaxFloat64
offset += int(unsafe.Sizeof(*msg.Field1))
msg.Field3 = (*test.NidOptNative)(unsafe.Pointer(&buf[offset]))
msg.Field3.Field1 = math.MaxFloat64 / 2
offset += int(unsafe.Sizeof(*msg.Field3))
msg.Field4 = (*test.NinOptNative)(unsafe.Pointer(&buf[offset]))
offset += int(unsafe.Sizeof(*msg.Field4))
msg.Field4.Field2 = (*float32)(unsafe.Pointer(&buf[offset]))
*msg.Field4.Field2 = math.MaxFloat32 / 3
offset += int(unsafe.Sizeof(*msg.Field4.Field2))
msg.Field6 = (*uint64)(unsafe.Pointer(&buf[offset]))
*msg.Field6 = math.MaxUint64 - 6
offset += int(unsafe.Sizeof(*msg.Field6))
t.Logf("%v", buf)
t.Logf("msg %#v", msg)
printWords(t, buf)
t.Logf("offset = %d offset/8 = %d", offset, offset/8)
if err := msg.VerboseEqual(structMax2); err != nil {
panic(err)
}
}
var (
structMax1 = &test.NinOptStruct{
Field1: proto.Float64(math.MaxFloat64),
Field3: &test.NidOptNative{},
}
)
func TestUnmarshalIntoNinOptStruct1(t *testing.T) {
var msg *test.NinOptStruct = nil
initSize := int(unsafe.Offsetof(msg.XXX_unrecognized) + unsafe.Sizeof(msg.XXX_unrecognized))
popSize := 1000
buf := make([]byte, initSize+popSize)
var offset int
msg = (*test.NinOptStruct)(unsafe.Pointer(&buf[0]))
offset = initSize
msg.Field1 = (*float64)(unsafe.Pointer(&buf[offset]))
*msg.Field1 = math.MaxFloat64
offset += int(unsafe.Sizeof(*msg.Field1))
msg.Field3 = (*test.NidOptNative)(unsafe.Pointer(&buf[offset]))
offset += int(unsafe.Sizeof(*msg.Field3))
t.Logf("%v", buf)
t.Logf("msg %#v", msg)
printWords(t, buf)
t.Logf("offset = %d offset/8 = %d", offset, offset/8)
if err := msg.VerboseEqual(structMax1); err != nil {
panic(err)
}
}
/*
message NinRepNative {
repeated double Field1 = 1;
repeated float Field2 = 2;
repeated int32 Field3 = 3;
repeated int64 Field4 = 4;
repeated uint32 Field5 = 5;
repeated uint64 Field6 = 6;
repeated sint32 Field7 = 7;
repeated sint64 Field8 = 8;
repeated fixed32 Field9 = 9;
repeated sfixed32 Field10 = 10;
repeated fixed64 Field11 = 11;
repeated sfixed64 Field12 = 12;
repeated bool Field13 = 13;
repeated string Field14 = 14;
repeated bytes Field15 = 15;
}
*/
var (
repMax = &test.NinRepNative{
Field1: []float64{math.MaxFloat64},
Field2: []float32{math.MaxFloat32, math.MaxFloat32 / 2},
Field3: []int32{math.MaxInt32, math.MaxInt32 / 2, math.MaxInt32 / 3},
Field14: []string{"ABCD", "CDEF"},
Field15: [][]byte{{1, 2, 3, 4}, {5, 6, 7}},
}
)
func TestUnmarshalIntoNinRepNative(t *testing.T) {
var msg *test.NinRepNative = nil
initSize := int(unsafe.Offsetof(msg.XXX_unrecognized) + unsafe.Sizeof(msg.XXX_unrecognized))
popSize := 1000
buf := make([]byte, initSize+popSize)
var offset int
msg = (*test.NinRepNative)(unsafe.Pointer(&buf[0]))
offset = initSize
v1 := []float64{math.MaxFloat64}
f1 := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field1))]))
f1.Data = uintptr(unsafe.Pointer(&buf[offset]))
f1.Len = 0
f1.Cap = len(v1)
for _, v := range v1 {
msg.Field1 = append(msg.Field1, v)
offset += int(unsafe.Sizeof(v))
}
v2 := []float32{math.MaxFloat32, math.MaxFloat32 / 2}
f2 := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field2))]))
f2.Data = uintptr(unsafe.Pointer(&buf[offset]))
f2.Len = 0
f2.Cap = len(v2)
for _, v := range v2 {
msg.Field2 = append(msg.Field2, v)
offset += int(unsafe.Sizeof(v))
}
v3 := []int32{math.MaxInt32, math.MaxInt32 / 2, math.MaxInt32 / 3}
f3 := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field3))]))
f3.Data = uintptr(unsafe.Pointer(&buf[offset]))
f3.Len = 0
f3.Cap = len(v3)
for _, v := range v3 {
msg.Field3 = append(msg.Field3, v)
offset += int(unsafe.Sizeof(v))
}
v14 := []string{"ABCD", "CDEF"}
f14 := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field14))]))
f14.Data = uintptr(unsafe.Pointer(&buf[offset]))
f14.Len = len(v14)
f14.Cap = len(v14)
n := len(v14) * 16
elem_offset := offset + n
for _, v := range v14 {
r := (*reflect.StringHeader)(unsafe.Pointer(&buf[offset]))
r.Data = uintptr(unsafe.Pointer(&buf[elem_offset]))
r.Len = len(v)
copy(buf[elem_offset:], v)
elem_offset += len(v)
offset += 16
}
offset = elem_offset
v15 := [][]byte{{1, 2, 3, 4}, {5, 6, 7}}
f15 := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field15))]))
f15.Data = uintptr(unsafe.Pointer(&buf[offset]))
f15.Len = len(v15)
f15.Cap = len(v15)
n15 := len(v15) * 24
elem_offset15 := offset + n15
for _, v := range v15 {
r := (*reflect.SliceHeader)(unsafe.Pointer(&buf[offset]))
r.Data = uintptr(unsafe.Pointer(&buf[elem_offset15]))
r.Len = len(v)
r.Cap = len(v)
copy(buf[elem_offset15:], v)
elem_offset15 += len(v)
offset += 24
}
t.Logf("%v", buf)
t.Logf("msg %#v", msg)
printWords(t, buf)
t.Logf("offset = %d offset/8 = %d", offset, offset/8)
if err := msg.VerboseEqual(repMax); err != nil {
panic(err)
}
}
/*
type NidOptNative struct {
Field1 float64 `protobuf:"fixed64,1,opt" json:"Field1"`
Field2 float32 `protobuf:"fixed32,2,opt" json:"Field2"`
Field3 int32 `protobuf:"varint,3,opt" json:"Field3"`
Field4 int64 `protobuf:"varint,4,opt" json:"Field4"`
Field5 uint32 `protobuf:"varint,5,opt" json:"Field5"`
Field6 uint64 `protobuf:"varint,6,opt" json:"Field6"`
Field7 int32 `protobuf:"zigzag32,7,opt" json:"Field7"`
Field8 int64 `protobuf:"zigzag64,8,opt" json:"Field8"`
Field9 uint32 `protobuf:"fixed32,9,opt" json:"Field9"`
Field10 int32 `protobuf:"fixed32,10,opt" json:"Field10"`
Field11 uint64 `protobuf:"fixed64,11,opt" json:"Field11"`
Field12 int64 `protobuf:"fixed64,12,opt" json:"Field12"`
Field13 bool `protobuf:"varint,13,opt" json:"Field13"`
Field14 string `protobuf:"bytes,14,opt" json:"Field14"`
Field15 []byte `protobuf:"bytes,15,opt" json:"Field15"`
XXX_unrecognized []byte `json:"-"`
}
*/
var (
nidMax = &test.NidOptNative{
Field1: math.MaxFloat64 - 1,
Field2: math.MaxFloat32 - 2,
Field3: math.MaxInt32 - 3,
Field4: math.MaxInt64 - 4,
Field5: math.MaxUint32 - 5,
Field6: math.MaxUint64 - 6,
Field7: math.MaxInt32 - 7,
Field8: math.MaxInt64 - 8,
Field9: math.MaxUint32 - 9,
Field10: math.MaxInt32 - 10,
Field11: math.MaxUint64 - 11,
Field12: math.MaxInt64 - 12,
Field13: true,
Field14: "ABCDEFGHI",
Field15: []byte{1, 2, 3, 5, 4, 5, 6, 7, 8},
XXX_unrecognized: []byte{255, 254, 253, 1, 252, 251, 250, 249, 248},
}
)
func TestUnmarshalIntoNidOptNative(t *testing.T) {
s := []byte("ABCDEFGHI")
data := []byte{1, 2, 3, 5, 4, 5, 6, 7, 8}
xxx := []byte{255, 254, 253, 1, 252, 251, 250, 249, 248}
popSize := len(s)
popSize += len(data)
popSize += len(xxx)
var msg *test.NidOptNative = nil
fieldPointerSize := int(unsafe.Offsetof(msg.XXX_unrecognized) + unsafe.Sizeof(msg.XXX_unrecognized))
//fieldPointerSize := (11 + 2 + 3 + 3) * 8
buf := make([]byte, fieldPointerSize+popSize)
var offset int
msg = (*test.NidOptNative)(unsafe.Pointer(&buf[0]))
msg.Field1 = math.MaxFloat64 - 1
msg.Field2 = math.MaxFloat32 - 2
msg.Field3 = math.MaxInt32 - 3
msg.Field4 = math.MaxInt64 - 4
msg.Field5 = math.MaxUint32 - 5
//see how 4 bytes are left empty to better align the bytes
msg.Field6 = math.MaxUint64 - 6
msg.Field7 = math.MaxInt32 - 7
//and again 4 bytes are left empty for alignment
msg.Field8 = math.MaxInt64 - 8
msg.Field9 = math.MaxUint32 - 9
msg.Field10 = math.MaxInt32 - 10
msg.Field11 = math.MaxUint64 - 11
msg.Field12 = math.MaxInt64 - 12
msg.Field13 = true
//the StringHeader is stored here
offset = fieldPointerSize
r := (*reflect.StringHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field14))]))
r.Data = uintptr(unsafe.Pointer(&buf[offset]))
r.Len = len(s)
copy(buf[offset:], s)
offset += len(s)
b := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field15))]))
b.Data = uintptr(unsafe.Pointer(&buf[offset]))
b.Len = len(data)
b.Cap = len(data)
copy(buf[offset:], data)
offset += len(data)
x := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.XXX_unrecognized))]))
x.Data = uintptr(unsafe.Pointer(&buf[offset]))
x.Len = len(xxx)
x.Cap = len(xxx)
copy(buf[offset:], xxx)
offset += len(xxx)
t.Logf("%v", buf)
t.Logf("msg %#v", msg)
printWords(t, buf)
t.Logf("offset = %d offset/8 = %d", offset, offset/8)
if err := msg.VerboseEqual(nidMax); err != nil {
panic(err)
}
}
/*
type NinOptNative struct {
Field1 *float64 `protobuf:"fixed64,1,opt" json:"Field1,omitempty"`
Field2 *float32 `protobuf:"fixed32,2,opt" json:"Field2,omitempty"`
Field3 *int32 `protobuf:"varint,3,opt" json:"Field3,omitempty"`
Field4 *int64 `protobuf:"varint,4,opt" json:"Field4,omitempty"`
Field5 *uint32 `protobuf:"varint,5,opt" json:"Field5,omitempty"`
Field6 *uint64 `protobuf:"varint,6,opt" json:"Field6,omitempty"`
Field7 *int32 `protobuf:"zigzag32,7,opt" json:"Field7,omitempty"`
Field8 *int64 `protobuf:"zigzag64,8,opt" json:"Field8,omitempty"`
Field9 *uint32 `protobuf:"fixed32,9,opt" json:"Field9,omitempty"`
Field10 *int32 `protobuf:"fixed32,10,opt" json:"Field10,omitempty"`
Field11 *uint64 `protobuf:"fixed64,11,opt" json:"Field11,omitempty"`
Field12 *int64 `protobuf:"fixed64,12,opt" json:"Field12,omitempty"`
Field13 *bool `protobuf:"varint,13,opt" json:"Field13,omitempty"`
Field14 *string `protobuf:"bytes,14,opt" json:"Field14,omitempty"`
Field15 []byte `protobuf:"bytes,15,opt" json:"Field15,omitempty"`
XXX_unrecognized []byte `json:"-"`
}
*/
var (
ninMax = &test.NinOptNative{
Field1: proto.Float64(math.MaxFloat64 - 1),
Field2: proto.Float32(math.MaxFloat32 - 2),
Field3: proto.Int32(math.MaxInt32 - 3),
Field4: proto.Int64(math.MaxInt64 - 4),
Field5: proto.Uint32(math.MaxUint32 - 5),
Field6: proto.Uint64(math.MaxUint64 - 6),
Field7: proto.Int32(math.MaxInt32 - 7),
Field8: proto.Int64(math.MaxInt64 - 8),
Field9: proto.Uint32(math.MaxUint32 - 9),
Field10: proto.Int32(math.MaxInt32 - 10),
Field11: proto.Uint64(math.MaxUint64 - 11),
Field12: proto.Int64(math.MaxInt64 - 12),
Field13: proto.Bool(true),
Field14: proto.String("ABCDEFGHI"),
Field15: []byte{1, 2, 3, 5, 4, 5, 6, 7, 8},
XXX_unrecognized: []byte{255, 254, 253, 1, 252, 251, 250, 249, 248},
}
)
//slice of bytes takes 3*8 bytes, one for pointer, one for len and one for cap, the sliceheader is stored inline.
//string takes 8 bytes, since this is only a pointer to the string header, unlike the slice header
//seems all other native pointers also take 8 bytes
func TestUnmarshalIntoNinOptNative(t *testing.T) {
s := []byte("ABCDEFGHI")
data := []byte{1, 2, 3, 5, 4, 5, 6, 7, 8}
xxx := []byte{255, 254, 253, 1, 252, 251, 250, 249, 248}
var msg *test.NinOptNative = nil
popSize := int(unsafe.Sizeof(*msg.Field1)) + int(unsafe.Sizeof(*msg.Field2)) + int(unsafe.Sizeof(*msg.Field3)) +
int(unsafe.Sizeof(*msg.Field4)) + int(unsafe.Sizeof(*msg.Field5)) + int(unsafe.Sizeof(*msg.Field6)) +
int(unsafe.Sizeof(*msg.Field7)) + int(unsafe.Sizeof(*msg.Field8)) + int(unsafe.Sizeof(*msg.Field9)) +
int(unsafe.Sizeof(*msg.Field10)) + int(unsafe.Sizeof(*msg.Field11)) + int(unsafe.Sizeof(*msg.Field12)) + int(unsafe.Sizeof(*msg.Field13))
popSize += 16 + len(s)
popSize += len(data)
popSize += len(xxx)
fieldPointerSize := int(unsafe.Offsetof(msg.XXX_unrecognized) + unsafe.Sizeof(msg.XXX_unrecognized))
//fieldPointerSize := (14 + 3 + 3) * 8
buf := make([]byte, fieldPointerSize+popSize)
var offset int
msg = (*test.NinOptNative)(unsafe.Pointer(&buf[0]))
offset = fieldPointerSize
msg.Field1 = (*float64)(unsafe.Pointer(&buf[offset]))
*msg.Field1 = math.MaxFloat64 - 1
offset += int(unsafe.Sizeof(*msg.Field1))
msg.Field2 = (*float32)(unsafe.Pointer(&buf[offset]))
*msg.Field2 = math.MaxFloat32 - 2
offset += int(unsafe.Sizeof(*msg.Field2))
msg.Field3 = (*int32)(unsafe.Pointer(&buf[offset]))
*msg.Field3 = math.MaxInt32 - 3
offset += int(unsafe.Sizeof(*msg.Field3))
msg.Field4 = (*int64)(unsafe.Pointer(&buf[offset]))
*msg.Field4 = math.MaxInt64 - 4
offset += int(unsafe.Sizeof(*msg.Field4))
msg.Field5 = (*uint32)(unsafe.Pointer(&buf[offset]))
*msg.Field5 = math.MaxUint32 - 5
offset += int(unsafe.Sizeof(*msg.Field5))
msg.Field6 = (*uint64)(unsafe.Pointer(&buf[offset]))
*msg.Field6 = math.MaxUint64 - 6
offset += int(unsafe.Sizeof(*msg.Field6))
msg.Field7 = (*int32)(unsafe.Pointer(&buf[offset]))
*msg.Field7 = math.MaxInt32 - 7
offset += int(unsafe.Sizeof(*msg.Field7))
msg.Field8 = (*int64)(unsafe.Pointer(&buf[offset]))
*msg.Field8 = math.MaxInt64 - 8
offset += int(unsafe.Sizeof(*msg.Field8))
msg.Field9 = (*uint32)(unsafe.Pointer(&buf[offset]))
*msg.Field9 = math.MaxUint32 - 9
offset += int(unsafe.Sizeof(*msg.Field9))
msg.Field10 = (*int32)(unsafe.Pointer(&buf[offset]))
*msg.Field10 = math.MaxInt32 - 10
offset += int(unsafe.Sizeof(*msg.Field10))
msg.Field11 = (*uint64)(unsafe.Pointer(&buf[offset]))
*msg.Field11 = math.MaxUint64 - 11
offset += int(unsafe.Sizeof(*msg.Field11))
msg.Field12 = (*int64)(unsafe.Pointer(&buf[offset]))
*msg.Field12 = math.MaxInt64 - 12
offset += int(unsafe.Sizeof(*msg.Field12))
msg.Field13 = (*bool)(unsafe.Pointer(&buf[offset]))
*msg.Field13 = true
offset += int(unsafe.Sizeof(*msg.Field13))
//offset += 8 //this might smaller than eight, for example one byte seems enough
//stores a string header here, which contains a pointer and a length = 16 bytes
msg.Field14 = (*string)(unsafe.Pointer(&buf[offset]))
//access the string header
r := (*reflect.StringHeader)(unsafe.Pointer(&buf[offset]))
offset += int(unsafe.Sizeof(*msg.Field14))
//set the pointer to the string
r.Data = uintptr(unsafe.Pointer(&buf[offset]))
//set the string length
r.Len = len(s)
//copy the string into the buffer
copy(buf[offset:], s)
offset += len(s)
msg.Field15 = *(*[]byte)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field15))]))
b := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.Field15))]))
b.Data = uintptr(unsafe.Pointer(&buf[offset]))
b.Len = len(data)
b.Cap = b.Len
copy(buf[offset:], data)
offset += len(data)
msg.XXX_unrecognized = *(*[]byte)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.XXX_unrecognized))]))
x := (*reflect.SliceHeader)(unsafe.Pointer(&buf[int(unsafe.Offsetof(msg.XXX_unrecognized))]))
x.Data = uintptr(unsafe.Pointer(&buf[offset]))
x.Len = len(xxx)
x.Cap = b.Len
copy(buf[offset:], xxx)
offset += len(xxx)
t.Logf("%v", buf)
t.Logf("msg %#v", msg)
printWords(t, buf)
t.Logf("offset = %d offset/8 = %d", offset, offset/8)
if err := msg.VerboseEqual(ninMax); err != nil {
panic(err)
}
}
func printWords(t *testing.T, buf []byte) {
zeros := []byte{0, 0, 0, 0, 0, 0, 0, 0}
size := len(buf) / 8
for i := 0; i < size; i++ {
if !bytes.Equal(buf[i*8:(i+1)*8], zeros) {
t.Logf("%d:%v", i+1, buf[i*8:(i+1)*8])
}
}
t.Logf("last:%v", buf[size*8:])
}