From 857f04efe306e7500a8d73f7475fe3315852cb2b Mon Sep 17 00:00:00 2001 From: Noah Metz Date: Thu, 31 Aug 2023 22:31:29 -0600 Subject: [PATCH] Ch-ch-ch-ch-changes --- context.go | 279 ++++++++++++++++++++++++++++++----------------- gql_test.go | 12 +- graph_test.go | 6 +- group.go | 2 +- lockable_test.go | 15 ++- node.go | 19 ++-- node_test.go | 9 +- 7 files changed, 217 insertions(+), 125 deletions(-) diff --git a/context.go b/context.go index c313383..96a3e57 100644 --- a/context.go +++ b/context.go @@ -43,7 +43,9 @@ func NewPolicyType(name string) PolicyType { } func NewSerializedType(name string) SerializedType { - return SerializedType(Hash(SerializedTypeBase, name)) + val := SerializedType(Hash(SerializedTypeBase, name)) + println(fmt.Sprintf("TYPE: %s: %d", name, val)) + return val } const ( @@ -89,6 +91,7 @@ var ( ExtensionType = NewSerializedType("extension") StringType = NewSerializedType("string") + Uint8Type = NewSerializedType("uint8") NodeKeyType = NewSerializedType("node_key") NodeNotFoundError = errors.New("Node not found in DB") @@ -112,18 +115,6 @@ type TypeInfo struct { Deserialize TypeDeserialize } -type Int int -func (i Int) MarshalBinary() ([]byte, error) { - ret := make([]byte, 8) - binary.BigEndian.PutUint64(ret, uint64(i)) - return ret, nil -} - -type String string -func (str String) MarshalBinary() ([]byte, error) { - return []byte(str), nil -} - // A Context stores all the data to run a graphvent process type Context struct { // DB is the database connection used to load and write nodes @@ -328,124 +319,194 @@ type SerializedValue struct { Data []byte } -func (field SerializedValue) MarshalBinary() ([]byte, error) { - data := []byte{} - for _, t := range(field.TypeStack) { - t_ser := make([]byte, 8) - binary.BigEndian.PutUint64(t_ser, uint64(t)) - data = append(data, t_ser...) - } - data = append(data, field.Data...) - return data, nil +func SerializeValue(ctx *Context, value reflect.Value) (SerializedValue, error) { + val, err := serializeValue(ctx, value.Type(), &value) + ctx.Log.Logf("serialize", "SERIALIZED_VALUE(%+v): %+v - %s", value, val, err) + return val, err } -func RecurseTypes(ctx *Context, t reflect.Type) ([]uint64, []reflect.Kind, error) { +func serializeValue(ctx *Context, t reflect.Type, value *reflect.Value) (SerializedValue, error) { var ctx_type uint64 = 0x00 ctype, exists := ctx.TypeReflects[t] + ctx.Log.Logf("serialize", "TYPE_REFLECTS: %+v", ctx.TypeReflects) if exists == true { + type_info := ctx.Types[ctype] ctx_type = uint64(ctype) + val_ser, err := type_info.Serialize(ctx, value.Interface()) + if err != nil { + return SerializedValue{}, err + } + return SerializedValue{ + []uint64{ctx_type}, + val_ser, + }, nil } - var new_types []uint64 - var new_kinds []reflect.Kind kind := t.Kind() switch kind { - case reflect.Array: - if ctx_type == 0x00 { - ctx_type = uint64(ArrayType) - } - elem_types, elem_kinds, err := RecurseTypes(ctx, t.Elem()) - if err != nil { - return nil, nil, err - } - new_types = append(new_types, ctx_type) - new_types = append(new_types, elem_types...) - - new_kinds = append(new_kinds, reflect.Array) - new_kinds = append(new_kinds, elem_kinds...) case reflect.Map: if ctx_type == 0x00 { ctx_type = uint64(MapType) } - key_types, key_kinds, err := RecurseTypes(ctx, t.Key()) + var data []byte + if value == nil { + data = nil + } else if value.IsZero() { + data = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} + } else if value.Len() == 0 { + data = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + } else { + map_iter := value.MapRange() + key_data := []byte{} + val_data := []byte{} + var key_types []uint64 = nil + var val_types []uint64 = nil + map_len := 0 + for map_iter.Next() { + map_len += 1 + key_value := map_iter.Key() + val_value := map_iter.Value() + + key, err := serializeValue(ctx, t.Key(), &key_value) + if err != nil { + return SerializedValue{}, err + } + val, err := serializeValue(ctx, t.Elem(), &val_value) + if err != nil { + return SerializedValue{}, err + } + + if key_types == nil { + key_types = key.TypeStack + val_types = val.TypeStack + } + + key_data = append(key_data, key.Data...) + val_data = append(val_data, val.Data...) + } + + type_stack := []uint64{ctx_type} + type_stack = append(type_stack, key_types...) + type_stack = append(type_stack, val_types...) + + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, uint64(map_len)) + data = append(data, key_data...) + data = append(data, val_data...) + return SerializedValue{ + type_stack, + data, + }, nil + } + key, err := serializeValue(ctx, t.Key(), nil) if err != nil { - return nil, nil, err + return SerializedValue{}, err } - elem_types, elem_kinds, err := RecurseTypes(ctx, t.Elem()) + elem, err := serializeValue(ctx, t.Elem(), nil) if err != nil { - return nil, nil, err + return SerializedValue{}, err } - new_types = append(new_types, ctx_type) - new_types = append(new_types, key_types...) - new_types = append(new_types, elem_types...) - - new_kinds = append(new_kinds, reflect.Map) - new_kinds = append(new_kinds, key_kinds...) - new_kinds = append(new_kinds, elem_kinds...) + type_stack := []uint64{ctx_type} + type_stack = append(type_stack, key.TypeStack...) + type_stack = append(type_stack, elem.TypeStack...) + return SerializedValue{ + type_stack, + data, + }, nil case reflect.Slice: if ctx_type == 0x00 { ctx_type = uint64(SliceType) } - elem_types, elem_kinds, err := RecurseTypes(ctx, t.Elem()) + var data []byte + if value == nil { + data = nil + } else if value.IsZero() { + data = []byte{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF} + } else if value.Len() == 0 { + data = []byte{0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00} + } else { + data := make([]byte, 8) + binary.BigEndian.PutUint64(data, uint64(value.Len())) + var elem SerializedValue + for i := 0; i < value.Len(); i += 1 { + val := value.Index(i) + element, err := serializeValue(ctx, t.Elem(), &val) + if err != nil { + return SerializedValue{}, err + } + if i == 0 { + elem = element + } + data = append(data, elem.Data...) + } + return SerializedValue{ + append([]uint64{ctx_type}, elem.TypeStack...), + data, + }, nil + } + elem, err := serializeValue(ctx, t.Elem(), nil) if err != nil { - return nil, nil, err + return SerializedValue{}, err } - new_types = append(new_types, ctx_type) - new_types = append(new_types, elem_types...) - - new_kinds = append(new_kinds, reflect.Slice) - new_kinds = append(new_kinds, elem_kinds...) + return SerializedValue{ + append([]uint64{ctx_type}, elem.TypeStack...), + data, + }, nil case reflect.Pointer: if ctx_type == 0x00 { ctx_type = uint64(PointerType) } - elem_types, elem_kinds, err := RecurseTypes(ctx, t.Elem()) + var data []byte + var elem_value *reflect.Value = nil + if value == nil { + data = nil + } else if value.IsZero() { + data = []byte{0x01} + } else { + data = []byte{0x00} + ev := value.Elem() + elem_value = &ev + } + elem, err := serializeValue(ctx, t.Elem(), elem_value) if err != nil { - return nil, nil, err + return SerializedValue{}, err } - new_types = append(new_types, ctx_type) - new_types = append(new_types, elem_types...) - - new_kinds = append(new_kinds, reflect.Pointer) - new_kinds = append(new_kinds, elem_kinds...) + if elem.Data != nil { + data = append(data, elem.Data...) + } + return SerializedValue{ + append([]uint64{uint64(ctx_type)}, elem.TypeStack...), + data, + }, nil case reflect.String: if ctx_type == 0x00 { ctx_type = uint64(StringType) } - new_types = append(new_types, ctx_type) - new_kinds = append(new_kinds, reflect.String) - default: - return nil, nil, fmt.Errorf("unhandled kind: %+v - %+v", kind, t) - } - return new_types, new_kinds, nil -} + if value == nil { + return SerializedValue{ + []uint64{ctx_type}, + nil, + }, nil + } -func serializeValue(ctx *Context, kind_stack []reflect.Kind, value reflect.Value) ([]byte, error) { - kind := kind_stack[len(kind_stack) - 1] - switch kind { + data := make([]byte, 8) + str := value.String() + binary.BigEndian.PutUint64(data, uint64(len(str))) + return SerializedValue{ + []uint64{uint64(ctx_type)}, + append(data, []byte(str)...), + }, nil + case reflect.Uint8: + if ctx_type == 0x00 { + ctx_type = uint64(Uint8Type) + } + return SerializedValue{ + []uint64{uint64(ctx_type)}, + []byte{uint8(value.Uint())}, + }, nil default: - return nil, fmt.Errorf("unhandled kind: %+v", kind) - } -} - -func SerializeValue(ctx *Context, value reflect.Value) (SerializedValue, error) { - if value.IsValid() == false { - return SerializedValue{}, fmt.Errorf("Cannot serialize invalid value: %+v", value) - } - - type_stack, kind_stack, err := RecurseTypes(ctx, value.Type()) - if err != nil { - return SerializedValue{}, err - } - - bytes, err := serializeValue(ctx, kind_stack, value) - if err != nil { - return SerializedValue{}, err + return SerializedValue{}, fmt.Errorf("unhandled kind: %+v - %+v", kind, t) } - return SerializedValue{ - type_stack, - bytes, - }, nil } /* @@ -532,13 +593,16 @@ func SerializeExtension(ctx *Context, ext Extension, ctx_type ExtType) (Serializ }, nil } +func (value SerializedValue) MarshalBinary() ([]byte, error) { + return nil, fmt.Errorf("SerializedValue.MarshalBinary Undefined") +} + +func ParseSerializedValue(ctx *Context, data []byte) (SerializedValue, []byte, error) { + return SerializedValue{}, nil, fmt.Errorf("ParseSerializedValue Undefined") +} + func DeserializeValue(ctx *Context, value SerializedValue) (interface{}, error) { - // TODO: do the opposite of SerializeValue. - // 1) Check the type to handle special types(array, list, map, pointer) - // 2) Check if the type is registered in the context, handle if so - // 3) Check if the type is a default type, handle if so - // 4) Return error if we don't know how to deserialize the type - return nil, fmt.Errorf("Undefined") + return nil, fmt.Errorf("DeserializeValue Undefined") } // Create a new Context with the base library content added @@ -559,6 +623,21 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { } var err error + err = ctx.RegisterType(reflect.TypeOf(SerializedValue{}), NewSerializedType("SerializedValue"), + func(ctx *Context, val interface{}) ([]byte, error) { + value := val.(SerializedValue) + return value.MarshalBinary() + }, func(ctx *Context, data []byte) (interface{}, error) { + value, data, err := ParseSerializedValue(ctx, data) + if err != nil { + return nil, err + } + if data != nil { + return nil, fmt.Errorf("%+v remaining after parse", data) + } + return value, nil + }) + err = ctx.RegisterExtension(reflect.TypeOf((*LockableExt)(nil)), LockableExtType, nil) if err != nil { return nil, err diff --git a/gql_test.go b/gql_test.go index f157db6..771b7dd 100644 --- a/gql_test.go +++ b/gql_test.go @@ -64,18 +64,20 @@ func TestGQLServer(t *testing.T) { fatalErr(t, err) listener_ext := NewListenerExt(10) - n1 := NewNode(ctx, nil, TestNodeType, 10, map[PolicyType]Policy{ + n1, err := NewNode(ctx, nil, TestNodeType, 10, map[PolicyType]Policy{ MemberOfPolicyType: &user_policy_2, AllNodesPolicyType: &user_policy_1, }, NewLockableExt(nil)) + fatalErr(t, err) - gql := NewNode(ctx, gql_key, GQLNodeType, 10, map[PolicyType]Policy{ + gql, err := NewNode(ctx, gql_key, GQLNodeType, 10, map[PolicyType]Policy{ MemberOfPolicyType: &group_policy_2, AllNodesPolicyType: &group_policy_1, }, NewLockableExt([]NodeID{n1.ID}), gql_ext, NewGroupExt(map[NodeID]string{ n1.ID: "user", gql_id: "self", }), listener_ext) + fatalErr(t, err) ctx.Log.Logf("test", "GQL: %s", gql.ID) ctx.Log.Logf("test", "NODE: %s", n1.ID) @@ -213,17 +215,19 @@ func TestGQLDB(t *testing.T) { TestUserNodeType := NewNodeType("TEST_USER") err := ctx.RegisterNodeType(TestUserNodeType, []ExtType{}) fatalErr(t, err) - u1 := NewNode(ctx, nil, TestUserNodeType, 10, nil) + u1, err := NewNode(ctx, nil, TestUserNodeType, 10, nil) + fatalErr(t, err) ctx.Log.Logf("test", "U1_ID: %s", u1.ID) gql_ext, err := NewGQLExt(ctx, ":0", nil, nil) fatalErr(t, err) listener_ext := NewListenerExt(10) - gql := NewNode(ctx, nil, GQLNodeType, 10, nil, + gql, err := NewNode(ctx, nil, GQLNodeType, 10, nil, gql_ext, listener_ext, NewGroupExt(nil)) + fatalErr(t, err) ctx.Log.Logf("test", "GQL_ID: %s", gql.ID) msgs := Messages{} diff --git a/graph_test.go b/graph_test.go index 7481fd4..04fa6e6 100644 --- a/graph_test.go +++ b/graph_test.go @@ -8,9 +8,9 @@ import ( var SimpleListenerNodeType = NewNodeType("SIMPLE_LISTENER") -func NewSimpleListener(ctx *Context, buffer int) (*Node, *ListenerExt) { +func NewSimpleListener(ctx *Context, buffer int) (*Node, *ListenerExt, error) { listener_extension := NewListenerExt(buffer) - listener := NewNode(ctx, + listener, err := NewNode(ctx, nil, SimpleListenerNodeType, 10, @@ -18,7 +18,7 @@ func NewSimpleListener(ctx *Context, buffer int) (*Node, *ListenerExt) { listener_extension, NewLockableExt(nil)) - return listener, listener_extension + return listener, listener_extension, err } func logTestContext(t * testing.T, components []string) *Context { diff --git a/group.go b/group.go index 792c3b0..0237e48 100644 --- a/group.go +++ b/group.go @@ -5,7 +5,7 @@ import ( ) type GroupExt struct { - Members map[NodeID]string `json:"members"` + Members map[NodeID]string } func (ext *GroupExt) Type() ExtType { diff --git a/lockable_test.go b/lockable_test.go index 1a0242d..0eb7eed 100644 --- a/lockable_test.go +++ b/lockable_test.go @@ -28,19 +28,21 @@ func TestLink(t *testing.T) { }) l2_listener := NewListenerExt(10) - l2 := NewNode(ctx, nil, TestLockableType, 10, + l2, err := NewNode(ctx, nil, TestLockableType, 10, map[PolicyType]Policy{ PerNodePolicyType: &policy, }, l2_listener, NewLockableExt(nil), ) + fatalErr(t, err) l1_listener := NewListenerExt(10) - l1 := NewNode(ctx, l1_key, TestLockableType, 10, nil, + l1, err := NewNode(ctx, l1_key, TestLockableType, 10, nil, l1_listener, NewLockableExt(nil), ) + fatalErr(t, err) msgs := Messages{} msgs = msgs.Add(ctx, l1.ID, l1.Key, NewLinkSignal("add", l2.ID), l1.ID) @@ -75,12 +77,13 @@ func Test10KLink(t *testing.T) { }, }) NewLockable := func()(*Node) { - l := NewNode(ctx, nil, TestLockableType, 10, + l, err := NewNode(ctx, nil, TestLockableType, 10, map[PolicyType]Policy{ PerNodePolicyType: &child_policy, }, NewLockableExt(nil), ) + fatalErr(t, err) return l } @@ -95,13 +98,14 @@ func Test10KLink(t *testing.T) { uint64(LockSignalType): nil, }) listener := NewListenerExt(100000) - node := NewNode(ctx, listener_key, TestLockableType, 10000, + node, err := NewNode(ctx, listener_key, TestLockableType, 10000, map[PolicyType]Policy{ AllNodesPolicyType: &l_policy, }, listener, NewLockableExt(reqs), ) + fatalErr(t, err) ctx.Log.Logf("test", "CREATED_LISTENER") _, err = LockLockable(ctx, node, node.ID) @@ -128,13 +132,14 @@ func TestLock(t *testing.T) { NewLockable := func(reqs []NodeID)(*Node, *ListenerExt) { listener := NewListenerExt(100) - l := NewNode(ctx, nil, TestLockableType, 10, + l, err := NewNode(ctx, nil, TestLockableType, 10, map[PolicyType]Policy{ AllNodesPolicyType: &policy, }, listener, NewLockableExt(reqs), ) + fatalErr(t, err) return l, listener } diff --git a/node.go b/node.go index 3f0f1a7..aaa7c7c 100644 --- a/node.go +++ b/node.go @@ -562,6 +562,7 @@ func (node *Node) Serialize(ctx *Context) (SerializedValue, error) { if err != nil { return SerializedValue{}, err } + ctx.Log.Logf("serialize", "SERIALIZED_EXTENSION: %+v", ext_bytes) node_bytes = append(node_bytes, ext_bytes...) } @@ -581,13 +582,13 @@ func KeyID(pub ed25519.PublicKey) NodeID { // Create a new node in memory and start it's event loop // TODO: Change panics to errors -func NewNode(ctx *Context, key ed25519.PrivateKey, node_type NodeType, buffer_size uint32, policies map[PolicyType]Policy, extensions ...Extension) *Node { +func NewNode(ctx *Context, key ed25519.PrivateKey, node_type NodeType, buffer_size uint32, policies map[PolicyType]Policy, extensions ...Extension) (*Node, error) { var err error var public ed25519.PublicKey if key == nil { public, key, err = ed25519.GenerateKey(rand.Reader) if err != nil { - panic(err) + return nil, err } } else { public = key.Public().(ed25519.PublicKey) @@ -595,23 +596,23 @@ func NewNode(ctx *Context, key ed25519.PrivateKey, node_type NodeType, buffer_si id := KeyID(public) _, exists := ctx.Node(id) if exists == true { - panic("Attempted to create an existing node") + return nil, fmt.Errorf("Attempted to create an existing node") } def, exists := ctx.Nodes[node_type] if exists == false { - panic("Node type %s not registered in Context") + return nil, fmt.Errorf("Node type %+v not registered in Context", node_type) } ext_map := map[ExtType]Extension{} for _, ext := range(extensions) { ext_type, exists := ctx.ExtensionTypes[reflect.TypeOf(ext)] if exists == false { - panic(fmt.Sprintf("%+v is not a known Extension", reflect.TypeOf(ext))) + return nil, fmt.Errorf(fmt.Sprintf("%+v is not a known Extension", reflect.TypeOf(ext))) } _, exists = ext_map[ext_type] if exists == true { - panic("Cannot add the same extension to a node twice") + return nil, fmt.Errorf("Cannot add the same extension to a node twice") } ext_map[ext_type] = ext } @@ -619,7 +620,7 @@ func NewNode(ctx *Context, key ed25519.PrivateKey, node_type NodeType, buffer_si for _, required_ext := range(def.Extensions) { _, exists := ext_map[required_ext] if exists == false { - panic(fmt.Sprintf("%+v requires %+v", node_type, required_ext)) + return nil, fmt.Errorf(fmt.Sprintf("%+v requires %+v", node_type, required_ext)) } } @@ -657,14 +658,14 @@ func NewNode(ctx *Context, key ed25519.PrivateKey, node_type NodeType, buffer_si err = WriteNode(ctx, node) if err != nil { - panic(err) + return nil, err } node.Process(ctx, ZeroID, NewCreateSignal()) go runNode(ctx, node) - return node + return node, nil } // Write a node to the database diff --git a/node_test.go b/node_test.go index 6603c56..2e1f222 100644 --- a/node_test.go +++ b/node_test.go @@ -13,7 +13,8 @@ func TestNodeDB(t *testing.T) { err := ctx.RegisterNodeType(node_type, []ExtType{GroupExtType}) fatalErr(t, err) - node := NewNode(ctx, nil, node_type, 10, nil, NewGroupExt(nil), NewLockableExt(nil)) + node, err := NewNode(ctx, nil, node_type, 10, nil, NewGroupExt(nil), NewLockableExt(nil)) + fatalErr(t, err) ctx.nodeMap = map[NodeID]*Node{} _, err = ctx.getNode(node.ID) @@ -44,11 +45,13 @@ func TestNodeRead(t *testing.T) { }) n2_listener := NewListenerExt(10) - n2 := NewNode(ctx, n2_key, node_type, 10, nil, NewGroupExt(nil), n2_listener) + n2, err := NewNode(ctx, n2_key, node_type, 10, nil, NewGroupExt(nil), n2_listener) + fatalErr(t, err) - n1 := NewNode(ctx, n1_key, node_type, 10, map[PolicyType]Policy{ + n1, err := NewNode(ctx, n1_key, node_type, 10, map[PolicyType]Policy{ PerNodePolicyType: &n1_policy, }, NewGroupExt(nil)) + fatalErr(t, err) read_sig := NewReadSignal(map[ExtType][]string{ GroupExtType: {"members"},