Changed serialization to not allocate any memory, expects to be passed enough memory to serialize the type

context.go

@@ -26,7 +26,7 @@ var (
   ECDH = ecdh.X25519()
 )
 
-type SerializeFn func(ctx *Context, value reflect.Value) ([]byte, error)
+type SerializeFn func(ctx *Context, value reflect.Value, data []byte) (int, error)
 type DeserializeFn func(ctx *Context, data []byte) (reflect.Value, []byte, error)
 
 type NodeFieldInfo struct {
@@ -992,8 +992,9 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
   var err error
  
   err = RegisterScalar[NodeID](ctx, stringify, unstringify[NodeID], unstringifyAST[NodeID],
-  func(ctx *Context, value reflect.Value) ([]byte, error) {
+  func(ctx *Context, value reflect.Value, data []byte) (int, error) {
-    return value.Bytes(), nil
+    copy(data, value.Bytes())
+    return 16, nil
   }, func(ctx *Context, data []byte) (reflect.Value, []byte, error) {
     if len(data) < 16 {
       return reflect.Value{}, nil, fmt.Errorf("Not enough bytes to decode NodeID(got %d, want 16)", len(data))
@@ -1012,8 +1013,9 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
   }
 
   err = RegisterScalar[uuid.UUID](ctx, stringify, unstringify[uuid.UUID], unstringifyAST[uuid.UUID],
-  func(ctx *Context, value reflect.Value) ([]byte, error) {
+  func(ctx *Context, value reflect.Value, data []byte) (int, error) {
-    return value.Bytes(), nil
+    copy(data, value.Bytes())
+    return 16, nil
   }, func(ctx *Context, data []byte) (reflect.Value, []byte, error) {
     if len(data) < 16 {
       return reflect.Value{}, nil, fmt.Errorf("Not enough bytes to decode uuid.UUID(got %d, want 16)", len(data))

db.go

@@ -8,11 +8,15 @@ import (
 	badger "github.com/dgraph-io/badger/v3"
 )
 
+const NODE_BUFFER_SIZE = 1000000
+
 func WriteNodeInit(ctx *Context, node *Node) error {
   if node == nil {
     return fmt.Errorf("Cannot serialize nil *Node")
   }
 
+  buffer := [NODE_BUFFER_SIZE]byte{}
+
   return ctx.DB.Update(func(tx *badger.Txn) error {
     // Get the base key bytes
     id_ser, err := node.ID.MarshalBinary()
@@ -21,22 +25,22 @@ func WriteNodeInit(ctx *Context, node *Node) error {
     }
 
     // Write Node value
-    node_val, err := Serialize(ctx, node)
+    written, err := Serialize(ctx, node, buffer[:])
     if err != nil {
       return err
     }
-    err = tx.Set(id_ser, node_val)
+    err = tx.Set(id_ser, buffer[:written])
     if err != nil {
       return err
     }
     
     // Write empty signal queue
     sigqueue_id := append(id_ser, []byte(" - SIGQUEUE")...)
-    sigqueue_val, err := Serialize(ctx, node.SignalQueue)
+    written, err = Serialize(ctx, node.SignalQueue, buffer[:])
     if err != nil {
       return err
     }
-    err = tx.Set(sigqueue_id, sigqueue_val)
+    err = tx.Set(sigqueue_id, buffer[:written])
     if err != nil {
       return err
     }
@@ -46,12 +50,12 @@ func WriteNodeInit(ctx *Context, node *Node) error {
     for ext_type := range(node.Extensions) {
       ext_list = append(ext_list, ext_type)
     }
-    ext_list_val, err := Serialize(ctx, ext_list)
+    written, err = Serialize(ctx, ext_list, buffer[:])
     if err != nil {
       return err
     }
     ext_list_id := append(id_ser, []byte(" - EXTLIST")...)
-    err = tx.Set(ext_list_id, ext_list_val)
+    err = tx.Set(ext_list_id, buffer[:written])
     if err != nil {
       return err
     }
@@ -60,17 +64,19 @@ func WriteNodeInit(ctx *Context, node *Node) error {
     for ext_type, ext := range(node.Extensions) {
       // Write each extension's current value
       ext_id := binary.BigEndian.AppendUint64(id_ser, uint64(ext_type))
-      ext_ser, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem())
+      written, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem(), buffer[:])
       if err != nil {
         return err
       }
-      err = tx.Set(ext_id, ext_ser)
+      err = tx.Set(ext_id, buffer[:written])
     }
     return nil
   })
 }
 
 func WriteNodeChanges(ctx *Context, node *Node, changes map[ExtType]Changes) error {
+  buffer := [NODE_BUFFER_SIZE]byte{}
+
   return ctx.DB.Update(func(tx *badger.Txn) error {
     // Get the base key bytes
     id_ser, err := node.ID.MarshalBinary()
@@ -83,11 +89,11 @@ func WriteNodeChanges(ctx *Context, node *Node, changes map[ExtType]Changes) err
       node.writeSignalQueue = false
 
       sigqueue_id := append(id_ser, []byte(" - SIGQUEUE")...)
-      sigqueue_val, err := Serialize(ctx, node.SignalQueue)
+      written, err := Serialize(ctx, node.SignalQueue, buffer[:])
       if err != nil {
         return fmt.Errorf("SignalQueue Serialize Error: %+v, %w", node.SignalQueue, err)
       }
-      err = tx.Set(sigqueue_id, sigqueue_val)
+      err = tx.Set(sigqueue_id, buffer[:written])
       if err != nil {
         return fmt.Errorf("SignalQueue set error: %+v, %w", node.SignalQueue, err)
       }
@@ -101,12 +107,12 @@ func WriteNodeChanges(ctx *Context, node *Node, changes map[ExtType]Changes) err
         return fmt.Errorf("%s is not an extension in %s", ext_type, node.ID)
       }
       ext_id := binary.BigEndian.AppendUint64(id_ser, uint64(ext_type))
-      ext_ser, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem())
+      written, err := SerializeValue(ctx, reflect.ValueOf(ext).Elem(), buffer[:])
       if err != nil {
         return fmt.Errorf("Extension serialize err: %s, %w", reflect.TypeOf(ext), err)
       }
 
-      err = tx.Set(ext_id, ext_ser)
+      err = tx.Set(ext_id, buffer[:written])
       if err != nil {
         return fmt.Errorf("Extension set err: %s, %w", reflect.TypeOf(ext), err)
       }

serialize.go

@@ -77,49 +77,59 @@ func GetFieldTag(tag string) FieldTag {
   return FieldTag(Hash("GRAPHVENT_FIELD_TAG", tag))
 }
 
-func TypeStack(ctx *Context, t reflect.Type) ([]byte, error) {
+func TypeStack(ctx *Context, t reflect.Type, data []byte) (int, error) {
   info, registered := ctx.Types[t]
   if registered {
-    return binary.BigEndian.AppendUint64(nil, uint64(info.Serialized)), nil
+    binary.BigEndian.PutUint64(data, uint64(info.Serialized))
+    return 8, nil
   } else {
     switch t.Kind() {
     case reflect.Map:
-      key_stack, err := TypeStack(ctx, t.Key())
+      binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Map)))
+
+      key_written, err := TypeStack(ctx, t.Key(), data[8:])
       if err != nil {
-        return nil, err
+        return 0, err
       }
 
-      elem_stack, err := TypeStack(ctx, t.Elem())
+      elem_written, err := TypeStack(ctx, t.Elem(), data[8 + key_written:])
       if err != nil {
-        return nil, err
+        return 0, err
       }
 
-      return append(binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Map))), append(key_stack, elem_stack...)...), nil
+      return 8 + key_written + elem_written, nil
+
     case reflect.Pointer:
-      elem_stack, err := TypeStack(ctx, t.Elem())
+      binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Pointer)))
+
+      elem_written, err := TypeStack(ctx, t.Elem(), data[8:])
       if err != nil {
-        return nil, err
+        return 0, err
       }
 
-      return append(binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Pointer))), elem_stack...), nil
+      return 8 + elem_written, nil
     case reflect.Slice:
-      elem_stack, err := TypeStack(ctx, t.Elem())
+      binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Slice)))
+
+      elem_written, err := TypeStack(ctx, t.Elem(), data[8:])
       if err != nil {
-        return nil, err
+        return 0, err
       }
 
-      return append(binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Slice))), elem_stack...), nil
+      return 8 + elem_written, nil
     case reflect.Array:
-      elem_stack, err := TypeStack(ctx, t.Elem())
+      binary.BigEndian.PutUint64(data, uint64(SerializeType(reflect.Array)))
+      binary.BigEndian.PutUint64(data[8:], uint64(t.Len()))
+
+      elem_written, err := TypeStack(ctx, t.Elem(), data[16:])
       if err != nil {
-        return nil, err
+        return 0, err
       }
 
-      stack := binary.BigEndian.AppendUint64(nil, uint64(SerializeType(reflect.Array)))
+      return 16 + elem_written, nil
-      stack = binary.BigEndian.AppendUint64(stack, uint64(t.Len()))
+
-      return append(stack, elem_stack...), nil
     default:
-      return nil, fmt.Errorf("Hit %s, which is not a registered type", t.String())
+      return 0, fmt.Errorf("Hit %s, which is not a registered type", t.String())
     }
   }
 }
@@ -173,8 +183,8 @@ func UnwrapStack(ctx *Context, stack []byte) (reflect.Type, []byte, error) {
   }
 }
 
-func Serialize[T any](ctx *Context, value T) ([]byte, error) {
+func Serialize[T any](ctx *Context, value T, data []byte) (int, error) {
-  return SerializeValue(ctx, reflect.ValueOf(&value).Elem())
+  return SerializeValue(ctx, reflect.ValueOf(&value).Elem(), data)
 }
 
 func Deserialize[T any](ctx *Context, data []byte) (T, error) {
@@ -192,7 +202,7 @@ func Deserialize[T any](ctx *Context, data []byte) (T, error) {
   return value.Interface().(T), nil
 }
 
-func SerializeValue(ctx *Context, value reflect.Value) ([]byte, error) {
+func SerializeValue(ctx *Context, value reflect.Value, data []byte) (int, error) {
   var serialize SerializeFn = nil
 
   info, registered := ctx.Types[value.Type()]
@@ -204,148 +214,155 @@ func SerializeValue(ctx *Context, value reflect.Value) ([]byte, error) {
     switch value.Type().Kind() {
     case reflect.Bool:
       if value.Bool() {
-        return []byte{0xFF}, nil
+        data[0] = 0xFF
       } else {
-        return []byte{0x00}, nil
+        data[0] = 0x00
       }
+      return 1, nil
 
     case reflect.Int8:
-      return []byte{byte(value.Int())}, nil
+      data[0] = byte(value.Int())
+      return 1, nil
     case reflect.Int16:
-      return binary.BigEndian.AppendUint16(nil, uint16(value.Int())), nil
+      binary.BigEndian.PutUint16(data, uint16(value.Int()))
+      return 2, nil
     case reflect.Int32:
-      return binary.BigEndian.AppendUint32(nil, uint32(value.Int())), nil
+      binary.BigEndian.PutUint32(data, uint32(value.Int()))
+      return 4, nil
     case reflect.Int64:
       fallthrough
     case reflect.Int:
-      return binary.BigEndian.AppendUint64(nil, uint64(value.Int())), nil
+      binary.BigEndian.PutUint64(data, uint64(value.Int()))
+      return 8, nil
 
     case reflect.Uint8:
-      return []byte{byte(value.Uint())}, nil
+      data[0] = byte(value.Uint())
+      return 1, nil
     case reflect.Uint16:
-      return binary.BigEndian.AppendUint16(nil, uint16(value.Uint())), nil
+      binary.BigEndian.PutUint16(data, uint16(value.Uint()))
+      return 2, nil
     case reflect.Uint32:
-      return binary.BigEndian.AppendUint32(nil, uint32(value.Uint())), nil
+      binary.BigEndian.PutUint32(data, uint32(value.Uint()))
+      return 4, nil
     case reflect.Uint64:
       fallthrough
     case reflect.Uint:
-      return binary.BigEndian.AppendUint64(nil, value.Uint()), nil
+      binary.BigEndian.PutUint64(data, value.Uint())
+      return 8, nil
 
     case reflect.Float32:
-      return binary.BigEndian.AppendUint32(nil, math.Float32bits(float32(value.Float()))), nil
+      binary.BigEndian.PutUint32(data, math.Float32bits(float32(value.Float())))
+      return 4, nil
     case reflect.Float64:
-      return binary.BigEndian.AppendUint64(nil, math.Float64bits(value.Float())), nil
+      binary.BigEndian.PutUint64(data, math.Float64bits(value.Float()))
+      return 8, nil
 
     case reflect.String:
-      len_bytes := make([]byte, 8)
+      binary.BigEndian.PutUint64(data, uint64(value.Len()))
-      binary.BigEndian.PutUint64(len_bytes, uint64(value.Len()))
+      copy(data[8:], []byte(value.String()))
-      return append(len_bytes, []byte(value.String())...), nil
+      return 8 + value.Len(), nil
 
     case reflect.Pointer:
       if value.IsNil() {
-        return []byte{0x00}, nil
+        data[0] = 0x00
+        return 1, nil
       } else {
-        elem, err := SerializeValue(ctx, value.Elem())
+        data[0] = 0x01
+        written, err := SerializeValue(ctx, value.Elem(), data[1:])
         if err != nil {
-          return nil, err
+          return 0, err
         }
-
+        return 1 + written, nil
-        return append([]byte{0x01}, elem...), nil
       }
 
     case reflect.Slice:
       if value.IsNil() {
-        return []byte{0x00}, nil
+        data[0] = 0x00
+        return 8, nil
       } else {
-        len_bytes := make([]byte, 8)
+        data[0] = 0x01
-        binary.BigEndian.PutUint64(len_bytes, uint64(value.Len()))
+        binary.BigEndian.PutUint64(data[1:], uint64(value.Len()))
-
+        total_written := 0
-        data := []byte{}
         for i := 0; i < value.Len(); i++ {
-          elem, err := SerializeValue(ctx, value.Index(i))
+          written, err := SerializeValue(ctx, value.Index(i), data[9+total_written:])
           if err != nil {
-            return nil, err
+            return 0, err
           }
-
+          total_written += written
-          data = append(data, elem...)
         }
-
+        return 9 + total_written, nil
-        return append(len_bytes, data...), nil
       }
 
     case reflect.Array:
       data := []byte{}
+      total_written := 0
       for i := 0; i < value.Len(); i++ {
-        elem, err := SerializeValue(ctx, value.Index(i))
+        written, err := SerializeValue(ctx, value.Index(i), data[total_written:])
         if err != nil {
-          return nil, err
+          return 0, err
         }
-
+        total_written += written
-        data = append(data, elem...)
       }
-      return data, nil
+      return total_written, nil
 
     case reflect.Map:
-      len_bytes := make([]byte, 8)
+      binary.BigEndian.PutUint64(data, uint64(value.Len()))
-      binary.BigEndian.PutUint64(len_bytes, uint64(value.Len()))
 
-      data := []byte{}
       key := reflect.New(value.Type().Key()).Elem()
       val := reflect.New(value.Type().Elem()).Elem()
       iter := value.MapRange()
+      total_written := 0
       for iter.Next() {
         key.SetIterKey(iter)
         val.SetIterValue(iter)
 
-        k, err := SerializeValue(ctx, key)
+        k, err := SerializeValue(ctx, key, data[8+total_written:])
         if err != nil {
-          return nil, err
+          return 0, err
         }
-        data = append(data, k...)
+        total_written += k
 
-        v, err := SerializeValue(ctx, val) 
+        v, err := SerializeValue(ctx, val, data[8+total_written:]) 
         if err != nil {
-          return nil, err
+          return 0, err
         }
-
+        total_written += v
-        data = append(data, v...)
       }
-      return append(len_bytes, data...), nil
+      return 8 + total_written, nil
     
     case reflect.Struct:
       if registered == false {
-        return nil, fmt.Errorf("Cannot serialize unregistered struct %s", value.Type())
+        return 0, fmt.Errorf("Cannot serialize unregistered struct %s", value.Type())
       } else {
-        data := binary.BigEndian.AppendUint64(nil, uint64(len(info.Fields)))
+        binary.BigEndian.PutUint64(data, uint64(len(info.Fields)))
 
+        total_written := 0
         for field_tag, field_info := range(info.Fields) {
-          data = append(data, binary.BigEndian.AppendUint64(nil, uint64(field_tag))...)
+          binary.BigEndian.PutUint64(data[8+total_written:], uint64(field_tag))
-          field_bytes, err := SerializeValue(ctx, value.FieldByIndex(field_info.Index))
+          total_written += 8
+          written, err := SerializeValue(ctx, value.FieldByIndex(field_info.Index), data[8+total_written:])
           if err != nil {
-            return nil, err
+            return 0, err
           }
-
+          total_written += written
-          data = append(data, field_bytes...)
         }
-        return data, nil
+        return 8 + total_written, nil
       }
 
     case reflect.Interface:
-      data, err := TypeStack(ctx, value.Elem().Type())
+      type_written, err := TypeStack(ctx, value.Elem().Type(), data)
 
-      val_data, err := SerializeValue(ctx, value.Elem())
+      elem_written, err := SerializeValue(ctx, value.Elem(), data[type_written:])
       if err != nil {
-        return nil, err
+        return 0, err
       }
 
-      data = append(data, val_data...)
+      return type_written + elem_written, nil
-
-      return data, nil
     default:
-      return nil, fmt.Errorf("Don't know how to serialize %s", value.Type())
+      return 0, fmt.Errorf("Don't know how to serialize %s", value.Type())
     }
   } else {
-    return serialize(ctx, value)
+    return serialize(ctx, value, data)
   }
 }
 
@@ -449,6 +466,11 @@ func DeserializeValue(ctx *Context, data []byte, t reflect.Type) (reflect.Value,
       }
 
     case reflect.Slice:
+      nil_byte := data[0]
+      data = data[1:]
+      if nil_byte == 0x00 {
+        return reflect.New(t).Elem(), data, nil
+      } else {
         len_bytes, left := split(data, 8)
         length := int(binary.BigEndian.Uint64(len_bytes))
         value := reflect.MakeSlice(t, length, length)
@@ -462,6 +484,7 @@ func DeserializeValue(ctx *Context, data []byte, t reflect.Type) (reflect.Value,
           value.Index(i).Set(elem_value)
         }
         return value, left, nil
+      }
 
     case reflect.Array:
       value := reflect.New(t).Elem()

serialize_test.go

@@ -7,10 +7,13 @@ import (
 )
 
 func testTypeStack[T any](t *testing.T, ctx *Context) {
+  buffer := [1024]byte{}
   reflect_type := reflect.TypeFor[T]()
-  stack, err := TypeStack(ctx, reflect_type)
+  written, err := TypeStack(ctx, reflect_type, buffer[:])
   fatalErr(t, err)
 
+  stack := buffer[:written]
+
   ctx.Log.Logf("test", "TypeStack[%s]: %+v", reflect_type, stack)
 
   unwrapped_type, rest, err := UnwrapStack(ctx, stack)
@@ -41,9 +44,12 @@ func TestSerializeTypes(t *testing.T) {
 }
 
 func testSerializeCompare[T comparable](t *testing.T, ctx *Context, value T) {
-  serialized, err := Serialize(ctx, value) 
+  buffer := [1024]byte{}
+  written, err := Serialize(ctx, value, buffer[:]) 
   fatalErr(t, err)
 
+  serialized := buffer[:written]
+
   ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[T](), value, serialized)
 
   deserialized, err := Deserialize[T](ctx, serialized)
@@ -57,9 +63,12 @@ func testSerializeCompare[T comparable](t *testing.T, ctx *Context, value T) {
 }
 
 func testSerializeList[L []T, T comparable](t *testing.T, ctx *Context, value L) {
-  serialized, err := Serialize(ctx, value)
+  buffer := [1024]byte{}
+  written, err := Serialize(ctx, value, buffer[:])
   fatalErr(t, err)
 
+  serialized := buffer[:written]
+
   ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[L](), value, serialized)
 
   deserialized, err := Deserialize[L](ctx, serialized)
@@ -75,9 +84,13 @@ func testSerializeList[L []T, T comparable](t *testing.T, ctx *Context, value L)
 }
 
 func testSerializePointer[P interface {*T}, T comparable](t *testing.T, ctx *Context, value P) {
-  serialized, err := Serialize(ctx, value)
+  buffer := [1024]byte{}
+
+  written, err := Serialize(ctx, value, buffer[:])
   fatalErr(t, err)
 
+  serialized := buffer[:written]
+
   ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[P](), value, serialized)
 
   deserialized, err := Deserialize[P](ctx, serialized)
@@ -97,9 +110,12 @@ func testSerializePointer[P interface {*T}, T comparable](t *testing.T, ctx *Con
 }
 
 func testSerialize[T any](t *testing.T, ctx *Context, value T) {
-  serialized, err := Serialize(ctx, value)
+  buffer := [1024]byte{}
+  written, err := Serialize(ctx, value, buffer[:])
   fatalErr(t, err)
 
+  serialized := buffer[:written]
+
   ctx.Log.Logf("test", "Serialized Value[%s : %+v]: %+v", reflect.TypeFor[T](), value, serialized)
 
   deserialized, err := Deserialize[T](ctx, serialized)