From 66d5e3f260a1a95fb992260ac3a6fa74995b8c13 Mon Sep 17 00:00:00 2001 From: Noah Metz Date: Thu, 28 Mar 2024 20:23:22 -0700 Subject: [PATCH] Changed serialization to not allocate any memory, expects to be passed enough memory to serialize the type --- context.go | 12 +-- db.go | 30 ++++--- serialize.go | 215 +++++++++++++++++++++++++--------------------- serialize_test.go | 26 ++++-- 4 files changed, 165 insertions(+), 118 deletions(-) diff --git a/context.go b/context.go index 2882636..86767c1 100644 --- a/context.go +++ b/context.go @@ -26,7 +26,7 @@ var ( ECDH = ecdh.X25519() ) -type SerializeFn func(ctx *Context, value reflect.Value) ([]byte, error) +type SerializeFn func(ctx *Context, value reflect.Value, data []byte) (int, error) type DeserializeFn func(ctx *Context, data []byte) (reflect.Value, []byte, error) type NodeFieldInfo struct { @@ -992,8 +992,9 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { var err error err = RegisterScalar[NodeID](ctx, stringify, unstringify[NodeID], unstringifyAST[NodeID], - func(ctx *Context, value reflect.Value) ([]byte, error) { - return value.Bytes(), nil + func(ctx *Context, value reflect.Value, data []byte) (int, error) { + copy(data, value.Bytes()) + return 16, nil }, func(ctx *Context, data []byte) (reflect.Value, []byte, error) { if len(data) < 16 { return reflect.Value{}, nil, fmt.Errorf("Not enough bytes to decode NodeID(got %d, want 16)", len(data)) @@ -1012,8 +1013,9 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { } err = RegisterScalar[uuid.UUID](ctx, stringify, unstringify[uuid.UUID], unstringifyAST[uuid.UUID], - func(ctx *Context, value reflect.Value) ([]byte, error) { - return value.Bytes(), nil + func(ctx *Context, value reflect.Value, data []byte) (int, error) { + copy(data, value.Bytes()) + return 16, nil }, func(ctx *Context, data []byte) (reflect.Value, []byte, error) { if len(data) < 16 { return reflect.Value{}, nil, fmt.Errorf("Not enough bytes to decode uuid.UUID(got %d, want 16)", len(data)) diff --git a/db.go b/db.go index d53f209..910e76d 100644 --- a/db.go +++ b/db.go @@ -8,11 +8,15 @@ import ( badger "github.com/dgraph-io/badger/v3" ) +const NODE_BUFFER_SIZE = 1000000 + func WriteNodeInit(ctx *Context, node *Node) error { if node == nil { return fmt.Errorf("Cannot serialize nil *Node") } + buffer := [NODE_BUFFER_SIZE]byte{} + return ctx.DB.Update(func(tx *badger.Txn) error { // Get the base key bytes id_ser, err := node.ID.MarshalBinary() @@ -21,22 +25,22 @@ func WriteNodeInit(ctx *Context, node *Node) error { } // Write Node value - node_val, err := Serialize(ctx, node) + written, err := Serialize(ctx, node, buffer[:]) if err != nil { return err } - err = tx.Set(id_ser, node_val) + err = tx.Set(id_ser, buffer[:written]) if err != nil { return err } // Write empty signal queue sigqueue_id := append(id_ser, []byte(" - SIGQUEUE")...) - sigqueue_val, err := Serialize(ctx, node.SignalQueue) + written, err = Serialize(ctx, node.SignalQueue, buffer[:]) if err != nil { return err } - err = tx.Set(sigqueue_id, sigqueue_val) + err = tx.Set(sigqueue_id, buffer[:written]) if err != nil { return err } @@ -46,12 +50,12 @@ func WriteNodeInit(ctx *Context, node *Node) error { for ext_type := range(node.Extensions) { ext_list = append(ext_list, ext_type) } - ext_list_val, err := Serialize(ctx, ext_list) + written, err = Serialize(ctx, ext_list, buffer[:]) if err != nil { return err } ext_list_id := append(id_ser, []byte(" - EXTLIST")...) - err = tx.Set(ext_list_id, ext_list_val) + err = tx.Set(ext_list_id, buffer[:written]) if err != nil { return err } @@ -60,17 +64,19 @@ func WriteNodeInit(ctx *Context, node *Node) error { for ext_type, ext := range(node.Extensions) { // Write each extension's current value ext_id := binary.BigEndian.AppendUint64(id_ser, uint64(ext_type)) - ext_ser, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem()) + written, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem(), buffer[:]) if err != nil { return err } - err = tx.Set(ext_id, ext_ser) + err = tx.Set(ext_id, buffer[:written]) } return nil }) } func WriteNodeChanges(ctx *Context, node *Node, changes map[ExtType]Changes) error { + buffer := [NODE_BUFFER_SIZE]byte{} + return ctx.DB.Update(func(tx *badger.Txn) error { // Get the base key bytes id_ser, err := node.ID.MarshalBinary() @@ -83,11 +89,11 @@ func WriteNodeChanges(ctx *Context, node *Node, changes map[ExtType]Changes) err node.writeSignalQueue = false sigqueue_id := append(id_ser, []byte(" - SIGQUEUE")...) - sigqueue_val, err := Serialize(ctx, node.SignalQueue) + written, err := Serialize(ctx, node.SignalQueue, buffer[:]) if err != nil { return fmt.Errorf("SignalQueue Serialize Error: %+v, %w", node.SignalQueue, err) } - err = tx.Set(sigqueue_id, sigqueue_val) + err = tx.Set(sigqueue_id, buffer[:written]) if err != nil { return fmt.Errorf("SignalQueue set error: %+v, %w", node.SignalQueue, err) } @@ -101,12 +107,12 @@ func WriteNodeChanges(ctx *Context, node *Node, changes map[ExtType]Changes) err return fmt.Errorf("%s is not an extension in %s", ext_type, node.ID) } ext_id := binary.BigEndian.AppendUint64(id_ser, uint64(ext_type)) - ext_ser, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem()) + written, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem(), buffer[:]) if err != nil { return fmt.Errorf("Extension serialize err: %s, %w", reflect.TypeOf(ext), err) } - err = tx.Set(ext_id, ext_ser) + err = tx.Set(ext_id, buffer[:written]) if err != nil { return fmt.Errorf("Extension set err: %s, %w", reflect.TypeOf(ext), err) } diff --git a/serialize.go b/serialize.go index db65de9..ece705d 100644 --- a/serialize.go +++ b/serialize.go @@ -77,49 +77,59 @@ func GetFieldTag(tag string) FieldTag { return FieldTag(Hash("GRAPHVENT_FIELD_TAG", tag)) } -func TypeStack(ctx *Context, t reflect.Type) ([]byte, error) { +func TypeStack(ctx *Context, t reflect.Type, data []byte) (int, error) { info, registered := ctx.Types[t] if registered { - return binary.BigEndian.AppendUint64(nil, uint64(info.Serialized)), nil + binary.BigEndian.PutUint64(data, uint64(info.Serialized)) + return 8, nil } else { switch t.Kind() { case reflect.Map: - key_stack, err := TypeStack(ctx, t.Key()) + binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Map))) + + key_written, err := TypeStack(ctx, t.Key(), data[8:]) if err != nil { - return nil, err + return 0, err } - elem_stack, err := TypeStack(ctx, t.Elem()) + elem_written, err := TypeStack(ctx, t.Elem(), data[8 + key_written:]) if err != nil { - return nil, err + return 0, err } - return append(binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Map))), append(key_stack, elem_stack...)...), nil + return 8 + key_written + elem_written, nil + case reflect.Pointer: - elem_stack, err := TypeStack(ctx, t.Elem()) + binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Pointer))) + + elem_written, err := TypeStack(ctx, t.Elem(), data[8:]) if err != nil { - return nil, err + return 0, err } - return append(binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Pointer))), elem_stack...), nil + return 8 + elem_written, nil case reflect.Slice: - elem_stack, err := TypeStack(ctx, t.Elem()) + binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Slice))) + + elem_written, err := TypeStack(ctx, t.Elem(), data[8:]) if err != nil { - return nil, err + return 0, err } - return append(binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Slice))), elem_stack...), nil + return 8 + elem_written, nil case reflect.Array: - elem_stack, err := TypeStack(ctx, t.Elem()) + binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Array))) + binary.BigEndian.PutUint64(data[8:], uint64(t.Len())) + + elem_written, err := TypeStack(ctx, t.Elem(), data[16:]) if err != nil { - return nil, err + return 0, err } - stack := binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Array))) - stack = binary.BigEndian.AppendUint64(stack, uint64(t.Len())) - return append(stack, elem_stack...), nil + return 16 + elem_written, nil + default: - return nil, fmt.Errorf("Hit %s, which is not a registered type", t.String()) + return 0, fmt.Errorf("Hit %s, which is not a registered type", t.String()) } } } @@ -173,8 +183,8 @@ func UnwrapStack(ctx *Context, stack []byte) (reflect.Type, []byte, error) { } } -func Serialize[T any](ctx *Context, value T) ([]byte, error) { - return SerializeValue(ctx, reflect.ValueOf(&value).Elem()) +func Serialize[T any](ctx *Context, value T, data []byte) (int, error) { + return SerializeValue(ctx, reflect.ValueOf(&value).Elem(), data) } func Deserialize[T any](ctx *Context, data []byte) (T, error) { @@ -192,7 +202,7 @@ func Deserialize[T any](ctx *Context, data []byte) (T, error) { return value.Interface().(T), nil } -func SerializeValue(ctx *Context, value reflect.Value) ([]byte, error) { +func SerializeValue(ctx *Context, value reflect.Value, data []byte) (int, error) { var serialize SerializeFn = nil info, registered := ctx.Types[value.Type()] @@ -204,148 +214,155 @@ func SerializeValue(ctx *Context, value reflect.Value) ([]byte, error) { switch value.Type().Kind() { case reflect.Bool: if value.Bool() { - return []byte{0xFF}, nil + data[0] = 0xFF } else { - return []byte{0x00}, nil + data[0] = 0x00 } + return 1, nil case reflect.Int8: - return []byte{byte(value.Int())}, nil + data[0] = byte(value.Int()) + return 1, nil case reflect.Int16: - return binary.BigEndian.AppendUint16(nil, uint16(value.Int())), nil + binary.BigEndian.PutUint16(data, uint16(value.Int())) + return 2, nil case reflect.Int32: - return binary.BigEndian.AppendUint32(nil, uint32(value.Int())), nil + binary.BigEndian.PutUint32(data, uint32(value.Int())) + return 4, nil case reflect.Int64: fallthrough case reflect.Int: - return binary.BigEndian.AppendUint64(nil, uint64(value.Int())), nil + binary.BigEndian.PutUint64(data, uint64(value.Int())) + return 8, nil case reflect.Uint8: - return []byte{byte(value.Uint())}, nil + data[0] = byte(value.Uint()) + return 1, nil case reflect.Uint16: - return binary.BigEndian.AppendUint16(nil, uint16(value.Uint())), nil + binary.BigEndian.PutUint16(data, uint16(value.Uint())) + return 2, nil case reflect.Uint32: - return binary.BigEndian.AppendUint32(nil, uint32(value.Uint())), nil + binary.BigEndian.PutUint32(data, uint32(value.Uint())) + return 4, nil case reflect.Uint64: fallthrough case reflect.Uint: - return binary.BigEndian.AppendUint64(nil, value.Uint()), nil + binary.BigEndian.PutUint64(data, value.Uint()) + return 8, nil case reflect.Float32: - return binary.BigEndian.AppendUint32(nil, math.Float32bits(float32(value.Float()))), nil + binary.BigEndian.PutUint32(data, math.Float32bits(float32(value.Float()))) + return 4, nil case reflect.Float64: - return binary.BigEndian.AppendUint64(nil, math.Float64bits(value.Float())), nil + binary.BigEndian.PutUint64(data, math.Float64bits(value.Float())) + return 8, nil case reflect.String: - len_bytes := make([]byte, 8) - binary.BigEndian.PutUint64(len_bytes, uint64(value.Len())) - return append(len_bytes, []byte(value.String())...), nil + binary.BigEndian.PutUint64(data, uint64(value.Len())) + copy(data[8:], []byte(value.String())) + return 8 + value.Len(), nil case reflect.Pointer: if value.IsNil() { - return []byte{0x00}, nil + data[0] = 0x00 + return 1, nil } else { - elem, err := SerializeValue(ctx, value.Elem()) + data[0] = 0x01 + written, err := SerializeValue(ctx, value.Elem(), data[1:]) if err != nil { - return nil, err + return 0, err } - - return append([]byte{0x01}, elem...), nil + return 1 + written, nil } case reflect.Slice: if value.IsNil() { - return []byte{0x00}, nil + data[0] = 0x00 + return 8, nil } else { - len_bytes := make([]byte, 8) - binary.BigEndian.PutUint64(len_bytes, uint64(value.Len())) - - data := []byte{} + data[0] = 0x01 + binary.BigEndian.PutUint64(data[1:], uint64(value.Len())) + total_written := 0 for i := 0; i < value.Len(); i++ { - elem, err := SerializeValue(ctx, value.Index(i)) + written, err := SerializeValue(ctx, value.Index(i), data[9+total_written:]) if err != nil { - return nil, err + return 0, err } - - data = append(data, elem...) + total_written += written } - - return append(len_bytes, data...), nil + return 9 + total_written, nil } case reflect.Array: data := []byte{} + total_written := 0 for i := 0; i < value.Len(); i++ { - elem, err := SerializeValue(ctx, value.Index(i)) + written, err := SerializeValue(ctx, value.Index(i), data[total_written:]) if err != nil { - return nil, err + return 0, err } - - data = append(data, elem...) + total_written += written } - return data, nil + return total_written, nil case reflect.Map: - len_bytes := make([]byte, 8) - binary.BigEndian.PutUint64(len_bytes, uint64(value.Len())) + binary.BigEndian.PutUint64(data, uint64(value.Len())) - data := []byte{} key := reflect.New(value.Type().Key()).Elem() val := reflect.New(value.Type().Elem()).Elem() iter := value.MapRange() + total_written := 0 for iter.Next() { key.SetIterKey(iter) val.SetIterValue(iter) - k, err := SerializeValue(ctx, key) + k, err := SerializeValue(ctx, key, data[8+total_written:]) if err != nil { - return nil, err + return 0, err } - data = append(data, k...) + total_written += k - v, err := SerializeValue(ctx, val) + v, err := SerializeValue(ctx, val, data[8+total_written:]) if err != nil { - return nil, err + return 0, err } - - data = append(data, v...) + total_written += v } - return append(len_bytes, data...), nil + return 8 + total_written, nil case reflect.Struct: if registered == false { - return nil, fmt.Errorf("Cannot serialize unregistered struct %s", value.Type()) + return 0, fmt.Errorf("Cannot serialize unregistered struct %s", value.Type()) } else { - data := binary.BigEndian.AppendUint64(nil, uint64(len(info.Fields))) + binary.BigEndian.PutUint64(data, uint64(len(info.Fields))) + total_written := 0 for field_tag, field_info := range(info.Fields) { - data = append(data, binary.BigEndian.AppendUint64(nil, uint64(field_tag))...) - field_bytes, err := SerializeValue(ctx, value.FieldByIndex(field_info.Index)) + binary.BigEndian.PutUint64(data[8+total_written:], uint64(field_tag)) + total_written += 8 + written, err := SerializeValue(ctx, value.FieldByIndex(field_info.Index), data[8+total_written:]) if err != nil { - return nil, err + return 0, err } - - data = append(data, field_bytes...) + total_written += written } - return data, nil + return 8 + total_written, nil } case reflect.Interface: - data, err := TypeStack(ctx, value.Elem().Type()) + type_written, err := TypeStack(ctx, value.Elem().Type(), data) - val_data, err := SerializeValue(ctx, value.Elem()) + elem_written, err := SerializeValue(ctx, value.Elem(), data[type_written:]) if err != nil { - return nil, err + return 0, err } - data = append(data, val_data...) - - return data, nil + return type_written + elem_written, nil default: - return nil, fmt.Errorf("Don't know how to serialize %s", value.Type()) + return 0, fmt.Errorf("Don't know how to serialize %s", value.Type()) } } else { - return serialize(ctx, value) + return serialize(ctx, value, data) } } @@ -449,19 +466,25 @@ func DeserializeValue(ctx *Context, data []byte, t reflect.Type) (reflect.Value, } case reflect.Slice: - len_bytes, left := split(data, 8) - length := int(binary.BigEndian.Uint64(len_bytes)) - value := reflect.MakeSlice(t, length, length) - for i := 0; i < length; i++ { - var elem_value reflect.Value - var err error - elem_value, left, err = DeserializeValue(ctx, left, t.Elem()) - if err != nil { - return reflect.Value{}, nil, err + nil_byte := data[0] + data = data[1:] + if nil_byte == 0x00 { + return reflect.New(t).Elem(), data, nil + } else { + len_bytes, left := split(data, 8) + length := int(binary.BigEndian.Uint64(len_bytes)) + value := reflect.MakeSlice(t, length, length) + for i := 0; i < length; i++ { + var elem_value reflect.Value + var err error + elem_value, left, err = DeserializeValue(ctx, left, t.Elem()) + if err != nil { + return reflect.Value{}, nil, err + } + value.Index(i).Set(elem_value) } - value.Index(i).Set(elem_value) + return value, left, nil } - return value, left, nil case reflect.Array: value := reflect.New(t).Elem() diff --git a/serialize_test.go b/serialize_test.go index 7d2eccd..7552f50 100644 --- a/serialize_test.go +++ b/serialize_test.go @@ -7,10 +7,13 @@ import ( ) func testTypeStack[T any](t *testing.T, ctx *Context) { + buffer := [1024]byte{} reflect_type := reflect.TypeFor[T]() - stack, err := TypeStack(ctx, reflect_type) + written, err := TypeStack(ctx, reflect_type, buffer[:]) fatalErr(t, err) + stack := buffer[:written] + ctx.Log.Logf("test", "TypeStack[%s]: %+v", reflect_type, stack) unwrapped_type, rest, err := UnwrapStack(ctx, stack) @@ -41,9 +44,12 @@ func TestSerializeTypes(t *testing.T) { } func testSerializeCompare[T comparable](t *testing.T, ctx *Context, value T) { - serialized, err := Serialize(ctx, value) + buffer := [1024]byte{} + written, err := Serialize(ctx, value, buffer[:]) fatalErr(t, err) + serialized := buffer[:written] + ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[T](), value, serialized) deserialized, err := Deserialize[T](ctx, serialized) @@ -57,9 +63,12 @@ func testSerializeCompare[T comparable](t *testing.T, ctx *Context, value T) { } func testSerializeList[L []T, T comparable](t *testing.T, ctx *Context, value L) { - serialized, err := Serialize(ctx, value) + buffer := [1024]byte{} + written, err := Serialize(ctx, value, buffer[:]) fatalErr(t, err) + serialized := buffer[:written] + ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[L](), value, serialized) deserialized, err := Deserialize[L](ctx, serialized) @@ -75,9 +84,13 @@ func testSerializeList[L []T, T comparable](t *testing.T, ctx *Context, value L) } func testSerializePointer[P interface {*T}, T comparable](t *testing.T, ctx *Context, value P) { - serialized, err := Serialize(ctx, value) + buffer := [1024]byte{} + + written, err := Serialize(ctx, value, buffer[:]) fatalErr(t, err) + serialized := buffer[:written] + ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[P](), value, serialized) deserialized, err := Deserialize[P](ctx, serialized) @@ -97,9 +110,12 @@ func testSerializePointer[P interface {*T}, T comparable](t *testing.T, ctx *Con } func testSerialize[T any](t *testing.T, ctx *Context, value T) { - serialized, err := Serialize(ctx, value) + buffer := [1024]byte{} + written, err := Serialize(ctx, value, buffer[:]) fatalErr(t, err) + serialized := buffer[:written] + ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[T](), value, serialized) deserialized, err := Deserialize[T](ctx, serialized)