Moved serialization to serialize.go and removed n parameter from DeserializeValue

gql.go

@@ -569,7 +569,7 @@ func (ctx *GQLExtContext) RegisterField(gql_type graphql.Type, gql_name string,
         return nil, fmt.Errorf("%s is not in the fields of %+v in the result", acl_name, ext_type)
       }
 
-      if val_ser.TypeStack[0] == uint64(ErrorType) {
+      if val_ser.TypeStack[0] == ErrorType {
         return nil, fmt.Errorf(string(val_ser.Data))
       }
 

gql_test.go

@@ -31,33 +31,33 @@ func TestGQLServer(t *testing.T) {
   gql_id := KeyID(pub)
 
   group_policy_1 := NewAllNodesPolicy(Tree{
-    uint64(ReadSignalType): Tree{
+    SerializedType(ReadSignalType): Tree{
-      uint64(GroupExtType): Tree{
+      SerializedType(GroupExtType): Tree{
         Hash(FieldNameBase, "members"): Tree{},
       },
     },
-    uint64(ReadResultSignalType): nil,
+    SerializedType(ReadResultSignalType): nil,
-    uint64(ErrorSignalType): nil,
+    SerializedType(ErrorSignalType): nil,
   })
 
   group_policy_2 := NewMemberOfPolicy(map[NodeID]Tree{
     gql_id: Tree{
-      uint64(LinkSignalType): nil,
+      SerializedType(LinkSignalType): nil,
-      uint64(LockSignalType): nil,
+      SerializedType(LockSignalType): nil,
-      uint64(StatusSignalType): nil,
+      SerializedType(StatusSignalType): nil,
-      uint64(ReadSignalType): nil,
+      SerializedType(ReadSignalType): nil,
     },
   })
 
   user_policy_1 := NewAllNodesPolicy(Tree{
-    uint64(ReadResultSignalType): nil,
+    SerializedType(ReadResultSignalType): nil,
-    uint64(ErrorSignalType): nil,
+    SerializedType(ErrorSignalType): nil,
   })
 
   user_policy_2 := NewMemberOfPolicy(map[NodeID]Tree{
     gql_id: Tree{
-      uint64(LinkSignalType): nil,
+      SerializedType(LinkSignalType): nil,
-      uint64(ReadSignalType): nil,
+      SerializedType(ReadSignalType): nil,
     },
   })
 

lockable_test.go

@@ -73,7 +73,7 @@ func Test10KLink(t *testing.T) {
   listener_id := KeyID(l_pub)
   child_policy := NewPerNodePolicy(map[NodeID]Tree{
     listener_id: Tree{
-      uint64(LockSignalType): nil,
+      SerializedType(LockSignalType): nil,
     },
   })
   NewLockable := func()(*Node) {
@@ -95,7 +95,7 @@ func Test10KLink(t *testing.T) {
   ctx.Log.Logf("test", "CREATED_10K")
 
   l_policy := NewAllNodesPolicy(Tree{
-    uint64(LockSignalType): nil,
+    SerializedType(LockSignalType): nil,
   })
   listener := NewListenerExt(100000)
   node, err := NewNode(ctx, listener_key, TestLockableType, 10000,

node.go

@@ -206,7 +206,7 @@ func NewErrorField(fstring string, args ...interface{}) SerializedValue {
     panic(err)
   }
   return SerializedValue{
-    TypeStack: []uint64{uint64(ErrorType)},
+    TypeStack: []SerializedType{ErrorType},
     Data: str_ser,
   }
 }
@@ -570,9 +570,9 @@ func NewNode(ctx *Context, key ed25519.PrivateKey, node_type NodeType, buffer_si
   }
 
   default_policy := NewAllNodesPolicy(Tree{
-    uint64(ErrorSignalType): nil,
+    SerializedType(ErrorSignalType): nil,
-    uint64(ReadResultSignalType): nil,
+    SerializedType(ReadResultSignalType): nil,
-    uint64(StatusSignalType): nil,
+    SerializedType(StatusSignalType): nil,
   })
 
   all_nodes_policy, exists := policies[AllNodesPolicyType]
@@ -665,7 +665,7 @@ func LoadNode(ctx * Context, id NodeID) (*Node, error) {
   } else if len(remaining) != 0 {
     return nil, fmt.Errorf("%d bytes left after parsing node from DB", len(remaining))
   }
-  _, node_val, remaining_data, err := DeserializeValue(ctx, value, 1)
+  _, node_val, remaining_data, err := DeserializeValue(ctx, value)
   if err != nil {
     return nil, err
   }
@@ -677,7 +677,7 @@ func LoadNode(ctx * Context, id NodeID) (*Node, error) {
     return nil, fmt.Errorf("%d bytes left after desrializing *Node", len(remaining_data.Data))
   }
 
-  node, ok := node_val[0].Interface().(*Node)
+  node, ok := node_val.Interface().(*Node)
   if ok == false {
     return nil, fmt.Errorf("Deserialized %+v when expecting *Node", reflect.TypeOf(node_val).Elem())
   }

node_test.go

@@ -55,7 +55,7 @@ func TestNodeRead(t *testing.T) {
 
   n1_policy := NewPerNodePolicy(map[NodeID]Tree{
     n2_id: {
-      uint64(ReadSignalType): nil,
+      SerializedType(ReadSignalType): nil,
     },
   })
 

policy.go

@@ -64,12 +64,12 @@ func (policy *RequirementOfPolicy) ContinueAllows(ctx *Context, current PendingA
     return Deny
   }
 
-  _, reqs_if, _, err := DeserializeValue(ctx, reqs_ser, 1)
+  _, reqs_if, _, err := DeserializeValue(ctx, reqs_ser)
   if err != nil {
     return Deny
   }
 
-  requirements, ok := reqs_if[0].Interface().(map[NodeID]ReqState)
+  requirements, ok := reqs_if.Interface().(map[NodeID]ReqState)
   if ok == false {
     return Deny
   }
@@ -113,12 +113,12 @@ func (policy *MemberOfPolicy) ContinueAllows(ctx *Context, current PendingACL, s
     return Deny
   }
 
-  _, members_if, _, err := DeserializeValue(ctx, members_ser, 1)
+  _, members_if, _, err := DeserializeValue(ctx, members_ser)
   if err != nil {
     return Deny
   }
 
-  members, ok := members_if[0].Interface().(map[NodeID]string)
+  members, ok := members_if.Interface().(map[NodeID]string)
   if ok == false {
     return Deny
   }
@@ -249,7 +249,7 @@ func (policy *AllNodesPolicy) Copy() Policy {
   }
 }
 
-type Tree map[uint64]Tree
+type Tree map[SerializedType]Tree
 
 func (rule Tree) Allows(action Tree) RuleResult {
   // If the current rule is nil, it's a wildcard and any action being processed is allowed
@@ -311,6 +311,6 @@ func (policy *AllNodesPolicy) Type() PolicyType {
 }
 
 var DefaultPolicy = NewAllNodesPolicy(Tree{
-  uint64(ErrorSignalType): nil,
+  SerializedType(ErrorSignalType): nil,
-  uint64(ReadResultSignalType): nil,
+  SerializedType(ReadResultSignalType): nil,
 })

serialize.go

@@ -0,0 +1,227 @@
+package graphvent
+
+import (
+  "crypto/sha512"
+  "encoding/binary"
+  "fmt"
+  "reflect"
+)
+
+const (
+  TagBase = "GraphventTag"
+  ExtTypeBase = "ExtType"
+  NodeTypeBase = "NodeType"
+  SignalTypeBase = "SignalType"
+  PolicyTypeBase = "PolicyType"
+  SerializedTypeBase = "SerializedType"
+  FieldNameBase = "FieldName"
+)
+
+func Hash(base string, name string) SerializedType {
+  digest := append([]byte(base), 0x00)
+  digest = append(digest, []byte(name)...)
+  hash := sha512.Sum512(digest)
+  return SerializedType(binary.BigEndian.Uint64(hash[0:8]))
+}
+
+type SerializedType uint64
+type ExtType SerializedType
+type NodeType SerializedType
+type SignalType SerializedType
+type PolicyType SerializedType
+
+type TypeSerialize func(*Context,SerializedType,reflect.Type,*reflect.Value) (SerializedValue, error)
+type TypeDeserialize func(*Context,SerializedValue) (reflect.Type, *reflect.Value, SerializedValue, error)
+
+func NewExtType(name string) ExtType {
+  return ExtType(Hash(ExtTypeBase, name))
+}
+
+func NewNodeType(name string) NodeType {
+  return NodeType(Hash(NodeTypeBase, name))
+}
+
+func NewSignalType(name string) SignalType {
+  return SignalType(Hash(SignalTypeBase, name))
+}
+
+func NewPolicyType(name string) PolicyType {
+  return PolicyType(Hash(PolicyTypeBase, name))
+}
+
+func NewSerializedType(name string) SerializedType {
+  return Hash(SerializedTypeBase, name)
+}
+
+var (
+  ListenerExtType = NewExtType("LISTENER")
+  LockableExtType = NewExtType("LOCKABLE")
+  GQLExtType      = NewExtType("GQL")
+  GroupExtType    = NewExtType("GROUP")
+  ECDHExtType     = NewExtType("ECDH")
+
+  GQLNodeType = NewNodeType("GQL")
+
+  StopSignalType       = NewSignalType("STOP")
+  CreateSignalType     = NewSignalType("CREATE")
+  StartSignalType      = NewSignalType("START")
+  ErrorSignalType      = NewSignalType("ERROR")
+  StatusSignalType     = NewSignalType("STATUS")
+  LinkSignalType       = NewSignalType("LINK")
+  LockSignalType       = NewSignalType("LOCK")
+  ReadSignalType       = NewSignalType("READ")
+  ReadResultSignalType = NewSignalType("READ_RESULT")
+  ACLTimeoutSignalType = NewSignalType("ACL_TIMEOUT")
+
+  MemberOfPolicyType      = NewPolicyType("USER_OF")
+  RequirementOfPolicyType = NewPolicyType("REQUIEMENT_OF")
+  PerNodePolicyType       = NewPolicyType("PER_NODE")
+  AllNodesPolicyType      = NewPolicyType("ALL_NODES")
+
+  ErrorType = NewSerializedType("ERROR")
+  PointerType = NewSerializedType("POINTER")
+  SliceType = NewSerializedType("SLICE")
+)
+
+type SerializedValue struct {
+  TypeStack []SerializedType
+  Data []byte
+}
+
+func (value SerializedValue) PopType() (SerializedType, SerializedValue, error) {
+  if len(value.TypeStack) == 0 {
+    return SerializedType(0), value, fmt.Errorf("No elements in TypeStack")
+  }
+  ctx_type := value.TypeStack[0]
+  value.TypeStack = value.TypeStack[1:]
+  return ctx_type, value, nil
+}
+
+func (value SerializedValue) PopData(n int) ([]byte, SerializedValue, error) {
+  if len(value.Data) < n {
+    return nil, value, fmt.Errorf("Not enough data %d/%d", len(value.Data), n)
+  }
+  data := value.Data[0:n]
+  value.Data = value.Data[n:]
+
+  return data, value, nil
+}
+
+func SerializeValue(ctx *Context, value reflect.Value) (SerializedValue, error) {
+  val, err := serializeValue(ctx, value.Type(), &value)
+  ctx.Log.Logf("serialize", "SERIALIZED_VALUE(%+v): %+v - %+v", value.Type(), val.TypeStack, val.Data)
+  return val, err
+}
+
+func serializeValue(ctx *Context, t reflect.Type, value *reflect.Value) (SerializedValue, error) {
+  ctx_type, type_exists := ctx.TypeReflects[t]
+  if type_exists == true {
+    type_info := ctx.Types[ctx_type]
+    if type_info.Serialize != nil {
+      return type_info.Serialize(ctx, ctx_type, t, value)
+    }
+  }
+
+  kind := t.Kind()
+  kind_info, handled := ctx.Kinds[kind]
+  if handled == false {
+    return SerializedValue{}, fmt.Errorf("Don't know how to serialize kind %+v", kind)
+  } else if type_exists == false {
+    ctx_type = kind_info.Type
+  }
+
+  return kind_info.Serialize(ctx, ctx_type, t, value)
+
+}
+
+func SerializeField(ctx *Context, ext Extension, field_name string) (SerializedValue, error) {
+  if ext == nil {
+    return SerializedValue{}, fmt.Errorf("Cannot get fields on nil Extension")
+  }
+  ext_value := reflect.ValueOf(ext).Elem()
+  field := ext_value.FieldByName(field_name)
+  if field.IsValid() == false {
+    return SerializedValue{}, fmt.Errorf("%s is not a field in %+v", field_name, ext)
+  } else {
+    return SerializeValue(ctx, field)
+  }
+}
+
+func (value SerializedValue) MarshalBinary() ([]byte, error) {
+  data := make([]byte, value.SerializedSize())
+  binary.BigEndian.PutUint64(data[0:8], uint64(len(value.TypeStack)))
+  binary.BigEndian.PutUint64(data[8:16], uint64(len(value.Data)))
+
+  for i, t := range(value.TypeStack) {
+    type_start := (i+2)*8
+    type_end := (i+3)*8
+    binary.BigEndian.PutUint64(data[type_start:type_end], uint64(t))
+  }
+
+  return append(data, value.Data...), nil
+}
+
+func (value SerializedValue) SerializedSize() uint64 {
+  return uint64((len(value.TypeStack) + 2) * 8)
+}
+
+func ParseSerializedValue(data []byte) (SerializedValue, []byte, error) {
+  if len(data) < 8 {
+    return SerializedValue{}, nil, fmt.Errorf("SerializedValue required to have at least 8 bytes when serialized")
+  }
+  num_types := int(binary.BigEndian.Uint64(data[0:8]))
+  data_size := int(binary.BigEndian.Uint64(data[8:16]))
+  type_stack := make([]SerializedType, num_types)
+  for i := 0; i < num_types; i += 1 {
+    type_start := (i+2) * 8
+    type_end := (i+3) * 8
+    type_stack[i] = SerializedType(binary.BigEndian.Uint64(data[type_start:type_end]))
+  }
+
+  types_end := 8*(num_types + 2)
+  data_end := types_end + data_size
+  return SerializedValue{
+    type_stack,
+    data[types_end:data_end],
+  }, data[data_end:], nil
+}
+
+func DeserializeValue(ctx *Context, value SerializedValue) (reflect.Type, *reflect.Value, SerializedValue, error) {
+  ctx.Log.Logf("serialize", "DeserializeValue: %+v", value)
+
+  var deserialize TypeDeserialize = nil
+  var reflect_type reflect.Type = nil
+  var reflect_value *reflect.Value = nil
+
+  ctx_type, value, err := value.PopType()
+  if err != nil {
+    return nil, nil, value, err
+  }
+
+  type_info, exists := ctx.Types[SerializedType(ctx_type)]
+  if exists == true {
+    deserialize = type_info.Deserialize
+    reflect_type = type_info.Type
+  } else {
+    kind, exists := ctx.KindTypes[SerializedType(ctx_type)]
+    if exists == false {
+      return nil, nil, value, fmt.Errorf("Cannot deserialize 0x%x: unknown type/kind", ctx_type)
+    }
+    kind_info := ctx.Kinds[kind]
+    deserialize = kind_info.Deserialize
+  }
+
+  ctx.Log.Logf("serialize", "Deserializing: %d", ctx_type)
+
+  if value.Data == nil {
+    reflect_type, _, value, err = deserialize(ctx, value)
+  } else {
+    reflect_type, reflect_value, value, err = deserialize(ctx, value)
+  }
+  if err != nil {
+    return nil, nil, value, err
+  }
+
+  ctx.Log.Logf("serialize", "DeserializeValue: DONE %+v - %+v - %+v", value, reflect_type, reflect_value)
+  return reflect_type, reflect_value, value, nil
+}

serialize_test.go

@@ -27,13 +27,6 @@ func TestSerializeBasic(t *testing.T) {
   testSerializeSliceSlice[[][]int](t, ctx, [][]int{{123, 456, 789, 101112}, {3253, 2341, 735, 212}, {123, 51}, nil})
   testSerializeSliceSlice[[][]string](t, ctx, [][]string{{"123", "456", "789", "101112"}, {"3253", "2341", "735", "212"}, {"123", "51"}, nil})
 
-  testSerializeMap(t, ctx, map[int]int{
-    1: 2,
-    3: 4,
-    5: 6,
-    7: 8,
-  })
-
   testSerialize(t, ctx, struct{
     int `gv:"0"`
     string `gv:"1"`
@@ -105,7 +98,7 @@ func testSerialize[T any](t *testing.T, ctx *Context, val T) T {
     t.Fatal("Data remaining after deserializing value")
   }
 
-  val_type, deserialized_values, remaining_deserialize, err := DeserializeValue(ctx, val_parsed, 1)
+  val_type, deserialized_values, remaining_deserialize, err := DeserializeValue(ctx, val_parsed)
   fatalErr(t, err)
 
   if len(remaining_deserialize.Data) != 0 {
@@ -116,12 +109,10 @@ func testSerialize[T any](t *testing.T, ctx *Context, val T) T {
     t.Fatal(fmt.Sprintf("DeserializeValue returned wrong reflect.Type %+v - %+v", val_type, reflect.TypeOf(val)))
   } else if deserialized_values == nil {
     t.Fatal("DeserializeValue returned no []reflect.Value")
-  } else if len(deserialized_values) == 0 {
+  } else if deserialized_values == nil {
-    t.Fatal("DeserializeValue returned empty []reflect.Value")
+    t.Fatal("DeserializeValue returned nil *reflect.Value")
-  } else if len(deserialized_values) > 1 {
+  } else if deserialized_values.CanConvert(val_type) == false {
-    t.Fatal("DeserializeValue returned wrong length []reflect.Value")
-  } else if deserialized_values[0].CanConvert(val_type) == false {
     t.Fatal("DeserializeValue returned value that can't convert to original value")
   }
-  return deserialized_values[0].Interface().(T)
+  return deserialized_values.Interface().(T)
 }

signal.go

@@ -144,7 +144,7 @@ func (signal *CreateSignal) Header() *SignalHeader {
 }
 func (signal *CreateSignal) Permission() Tree {
   return Tree{
-    uint64(CreateSignalType): nil,
+    SerializedType(CreateSignalType): nil,
   }
 }
 
@@ -162,7 +162,7 @@ func (signal *StartSignal) Header() *SignalHeader {
 }
 func (signal *StartSignal) Permission() Tree {
   return Tree{
-    uint64(StartSignalType): nil,
+    SerializedType(StartSignalType): nil,
   }
 }
 func NewStartSignal() *StartSignal {
@@ -179,7 +179,7 @@ func (signal *StopSignal) Header() *SignalHeader {
 }
 func (signal *StopSignal) Permission() Tree {
   return Tree{
-    uint64(StopSignalType): nil,
+    SerializedType(StopSignalType): nil,
   }
 }
 func NewStopSignal() *StopSignal {
@@ -250,7 +250,7 @@ func (signal *ErrorSignal) Deserialize(ctx *Context, data []byte) error {
 }
 func (signal *ErrorSignal) Permission() Tree {
   return Tree{
-    uint64(ErrorSignalType): nil,
+    SerializedType(ErrorSignalType): nil,
   }
 }
 func NewErrorSignal(req_id uuid.UUID, fmt_string string, args ...interface{}) Signal {
@@ -268,7 +268,7 @@ func (signal *ACLTimeoutSignal) Header() *SignalHeader {
 }
 func (signal *ACLTimeoutSignal) Permission() Tree {
   return Tree{
-    uint64(ACLTimeoutSignalType): nil,
+    SerializedType(ACLTimeoutSignalType): nil,
   }
 }
 func NewACLTimeoutSignal(req_id uuid.UUID) *ACLTimeoutSignal {
@@ -288,7 +288,7 @@ func (signal *StatusSignal) Header() *SignalHeader {
 }
 func (signal *StatusSignal) Permission() Tree {
   return Tree{
-    uint64(StatusSignalType): nil,
+    SerializedType(StatusSignalType): nil,
   }
 }
 func NewStatusSignal(source NodeID, status string) *StatusSignal {
@@ -315,7 +315,7 @@ const (
 
 func (signal *LinkSignal) Permission() Tree {
   return Tree{
-    uint64(LinkSignalType): Tree{
+    SerializedType(LinkSignalType): Tree{
       Hash(LinkActionBase, signal.Action): nil,
     },
   }
@@ -342,7 +342,7 @@ const (
 
 func (signal *LockSignal) Permission() Tree {
   return Tree{
-    uint64(LockSignalType): Tree{
+    SerializedType(LockSignalType): Tree{
       Hash(LockStateBase, signal.State): nil,
     },
   }
@@ -418,9 +418,9 @@ func (signal *ReadSignal) Permission() Tree {
     for _, field := range(fields) {
       field_tree[Hash(FieldNameBase, field)]  = nil
     }
-    ret[uint64(ext)] = field_tree
+    ret[SerializedType(ext)] = field_tree
   }
-  return Tree{uint64(ReadSignalType): ret}
+  return Tree{SerializedType(ReadSignalType): ret}
 }
 func NewReadSignal(exts map[ExtType][]string) *ReadSignal {
   return &ReadSignal{
@@ -440,7 +440,7 @@ func (signal *ReadResultSignal) Header() *SignalHeader {
 }
 func (signal *ReadResultSignal) Permission() Tree {
   return Tree{
-    uint64(ReadResultSignalType): nil,
+    SerializedType(ReadResultSignalType): nil,
   }
 }
 func NewReadResultSignal(req_id uuid.UUID, node_id NodeID, node_type NodeType, exts map[ExtType]map[string]SerializedValue) *ReadResultSignal {