MetzNet
/
graphvent
4
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved int, struct, and interface serialization to functions to make registering types easy

gql_cataclysm
noah metz 2023-09-11 21:47:53 -06:00
parent d0f0fb1b82
commit 045304f9f6
4 changed files with 523 additions and 334 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

398
context.go
Unescape Escape View File

@ -174,6 +174,10 @@ func (ctx *Context)RegisterType(reflect_type reflect.Type, ctx_type SerializedTy
return fmt.Errorf("Cannot register field with type %+v, type already registered in context", reflect_type) return fmt.Errorf("Cannot register field with type %+v, type already registered in context", reflect_type)
} }
if serialize == nil || deserialize == nil {
return fmt.Errorf("Cannot register field without serialize/deserialize functions")
}
type_info := TypeInfo{ type_info := TypeInfo{
Reflect: reflect_type, Reflect: reflect_type,
Type: ctx_type, Type: ctx_type,
@ -183,6 +187,8 @@ func (ctx *Context)RegisterType(reflect_type reflect.Type, ctx_type SerializedTy
ctx.Types[ctx_type] = &type_info ctx.Types[ctx_type] = &type_info
ctx.TypeReflects[reflect_type] = &type_info ctx.TypeReflects[reflect_type] = &type_info
ctx.Log.Logf("serialize", "Registered Type: %+v - %+v", reflect_type, ctx_type)
return nil return nil
} }
@ -325,15 +331,12 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
return nil, err return nil, err
} }
// TODO: figure out why this doesn't break in the simple test, but breaks in TestGQLDB
err = ctx.RegisterKind(reflect.Struct, StructType, err = ctx.RegisterKind(reflect.Struct, StructType,
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
serialized_value := SerializedValue{ type_stack := []SerializedType{ctx_type}
[]SerializedType{ctx_type}, var data []byte
nil,
}
if value != nil { if value != nil {
serialized_value.Data = make([]byte, 8) data = make([]byte, 8)
} }
num_fields := uint64(0) num_fields := uint64(0)
@ -348,29 +351,31 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
field_hash := Hash(FieldNameBase, gv_tag) field_hash := Hash(FieldNameBase, gv_tag)
field_hash_bytes := make([]byte, 8) field_hash_bytes := make([]byte, 8)
binary.BigEndian.PutUint64(field_hash_bytes, uint64(field_hash)) binary.BigEndian.PutUint64(field_hash_bytes, uint64(field_hash))
if value == nil { if value != nil {
field_ser, err := SerializeValue(ctx, field.Type, nil) field_value := value.FieldByIndex(field.Index)
field_ser, err := SerializeValue(ctx, field.Type, &field_value)
if err != nil { if err != nil {
return SerializedValue{}, err return SerializedValue{}, err
} }
serialized_value.TypeStack = append(serialized_value.TypeStack, field_ser.TypeStack...)
} else { field_bytes, err := field_ser.MarshalBinary()
field_value := value.FieldByIndex(field.Index)
field_ser, err := SerializeValue(ctx, field.Type, &field_value)
if err != nil { if err != nil {
return SerializedValue{}, err return SerializedValue{}, err
} }
serialized_value.TypeStack = append(serialized_value.TypeStack, field_ser.TypeStack...) data = append(data, field_hash_bytes...)
serialized_value.Data = append(serialized_value.Data, field_hash_bytes...) data = append(data, field_bytes...)
serialized_value.Data = append(serialized_value.Data, field_ser.Data...)
} }
} }
} }
if value != nil { if value != nil {
binary.BigEndian.PutUint64(serialized_value.Data[0:8], num_fields) binary.BigEndian.PutUint64(data[0:8], num_fields)
} }
return serialized_value, nil return SerializedValue{
type_stack,
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil { if value.Data == nil {
return reflect.TypeOf(map[uint64]reflect.Value{}), nil, value, nil return reflect.TypeOf(map[uint64]reflect.Value{}), nil, value, nil
@ -390,52 +395,31 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
if num_fields == 0 { if num_fields == 0 {
return map_type, &map_value, value, nil return map_type, &map_value, value, nil
} else { } else {
tmp_value := value tmp_data := value.Data
for i := 0; i < num_fields; i += 1 { for i := 0; i < num_fields; i += 1 {
var field_hash_bytes []byte if len(tmp_data) < 8 {
field_hash_bytes, tmp_value, err = tmp_value.PopData(8) return nil, nil, value, fmt.Errorf("Not enough data to deserialize struct field")
}
field_hash := binary.BigEndian.Uint64(tmp_data[0:8])
tmp_data = tmp_data[8:]
var field_value SerializedValue
field_value, tmp_data, err = ParseSerializedValue(tmp_data)
if err != nil { if err != nil {
return nil, nil, value, err return nil, nil, value, err
} }
field_hash := binary.BigEndian.Uint64(field_hash_bytes)
field_hash_value := reflect.ValueOf(field_hash) field_hash_value := reflect.ValueOf(field_hash)
var elem_value *reflect.Value var elem_value *reflect.Value
_, elem_value, tmp_value, err = DeserializeValue(ctx, tmp_value) _, elem_value, _, err = DeserializeValue(ctx, field_value)
if err != nil { if err != nil {
return nil, nil, value, err return nil, nil, value, err
} }
map_value.SetMapIndex(field_hash_value, reflect.ValueOf(*elem_value)) map_value.SetMapIndex(field_hash_value, reflect.ValueOf(*elem_value))
} }
return map_type, &map_value, tmp_value, nil value.Data = tmp_data
} return map_type, &map_value, value, nil
}
})
if err != nil {
return nil, err
}
err = ctx.RegisterKind(reflect.Int, IntType,
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = make([]byte, 8)
binary.BigEndian.PutUint64(data, uint64(value.Int()))
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(0), nil, value, nil
} }
if len(value.Data) < 8 {
return nil, nil, SerializedValue{}, fmt.Errorf("invalid length: %d/8", len(value.Data))
} }
int_val := reflect.ValueOf(int(binary.BigEndian.Uint64(value.Data[0:8])))
value.Data = value.Data[8:]
return int_val.Type(), &int_val, value, nil
}) })
if err != nil { if err != nil {
return nil, err return nil, err
@ -544,34 +528,6 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Uint32, UInt32Type,
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
data := make([]byte, 4)
if value != nil {
binary.BigEndian.PutUint32(data, uint32(value.Uint()))
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(uint32(0)), nil, value, nil
} else {
if len(value.Data) < 4 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint32")
}
val := binary.BigEndian.Uint32(value.Data[0:4])
value.Data = value.Data[4:]
int_value := reflect.ValueOf(val)
return int_value.Type(), &int_value, value, nil
}
})
if err != nil {
return nil, err
}
err = ctx.RegisterKind(reflect.String, StringType, err = ctx.RegisterKind(reflect.String, StringType,
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
if value == nil { if value == nil {
@ -651,47 +607,7 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Interface, InterfaceType, err = ctx.RegisterKind(reflect.Interface, InterfaceType, SerializeInterface, DeserializeInterface[interface{}]())
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte
type_stack := []SerializedType{ctx_type}
if value == nil {
data = nil
} else if value.IsZero() {
} else {
elem_value := value.Elem()
elem, err := SerializeValue(ctx, elem_value.Type(), &elem_value)
if err != nil {
return SerializedValue{}, err
}
data, err = elem.MarshalBinary()
if err != nil {
return SerializedValue{}, err
}
}
return SerializedValue{
type_stack,
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf((interface{})(nil)), nil, value, nil
} else {
var elem_value *reflect.Value
var elem_ser SerializedValue
var elem_type reflect.Type
var err error
elem_ser, value.Data, err = ParseSerializedValue(value.Data)
elem_type, elem_value, _, err = DeserializeValue(ctx, elem_ser)
if err != nil {
return nil, nil, value, err
}
ptr_type := reflect.PointerTo(elem_type)
ptr_value := reflect.New(ptr_type).Elem()
ptr_value.Set(elem_value.Addr())
return ptr_type, &ptr_value, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -862,230 +778,52 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Int8, Int8Type, err = ctx.RegisterKind(reflect.Int8, Int8Type, SerializeIntN(1), DeserializeIntN[int8](1))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = []byte{byte(value.Int())}
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(int8(0)), nil, value, nil
} else {
if len(value.Data) < 1 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize int8")
}
i := int8(value.Data[0])
value.Data = value.Data[1:]
val := reflect.ValueOf(i)
return val.Type(), &val, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Uint8, UInt8Type, err = ctx.RegisterKind(reflect.Int16, Int16Type, SerializeIntN(2), DeserializeIntN[int16](2))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = []byte{uint8(value.Uint())}
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(uint8(0)), nil, value, nil
} else {
if len(value.Data) < 1 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint8")
}
i := uint8(value.Data[0])
value.Data = value.Data[1:]
val := reflect.ValueOf(i)
return val.Type(), &val, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Uint16, UInt16Type, err = ctx.RegisterKind(reflect.Int32, Int32Type, SerializeIntN(4), DeserializeIntN[int32](4))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = make([]byte, 2)
binary.BigEndian.PutUint16(data, uint16(value.Uint()))
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(uint16(0)), nil, value, nil
} else {
if len(value.Data) < 2 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint16")
}
val := binary.BigEndian.Uint16(value.Data[0:2])
value.Data = value.Data[2:]
i := reflect.ValueOf(val)
return i.Type(), &i, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Int16, Int16Type, err = ctx.RegisterKind(reflect.Int64, Int64Type, SerializeIntN(8), DeserializeIntN[int64](8))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = make([]byte, 2)
binary.BigEndian.PutUint16(data, uint16(value.Int()))
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(int16(0)), nil, value, nil
} else {
if len(value.Data) < 2 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint16")
}
val := int16(binary.BigEndian.Uint16(value.Data[0:2]))
value.Data = value.Data[2:]
i := reflect.ValueOf(val)
return i.Type(), &i, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Int32, Int32Type, err = ctx.RegisterKind(reflect.Int, IntType, SerializeIntN(8), DeserializeIntN[int](8))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = make([]byte, 4)
binary.BigEndian.PutUint32(data, uint32(value.Int()))
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(int32(0)), nil, value, nil
} else {
if len(value.Data) < 4 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint16")
}
val := int32(binary.BigEndian.Uint32(value.Data[0:4]))
value.Data = value.Data[4:]
i := reflect.ValueOf(val)
return i.Type(), &i, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Uint, UIntType, err = ctx.RegisterKind(reflect.Uint8, UInt8Type, SerializeUintN(1), DeserializeUintN[uint8](1))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){
var data []byte = nil
if value != nil {
data = make([]byte, 8)
binary.BigEndian.PutUint64(data, value.Uint())
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(uint(0)), nil, value, nil
} else {
if len(value.Data) < 8 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize SerializedType")
}
val := uint(binary.BigEndian.Uint64(value.Data[0:8]))
value.Data = value.Data[8:]
i := reflect.ValueOf(val)
return i.Type(), &i, value, nil
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Uint64, UInt64Type, err = ctx.RegisterKind(reflect.Uint16, UInt16Type, SerializeUintN(2), DeserializeUintN[uint16](2))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ if err != nil {
var data []byte = nil return nil, err
if value != nil {
data = make([]byte, 8)
binary.BigEndian.PutUint64(data, value.Uint())
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(SerializedType(0)), nil, value, nil
} else {
if len(value.Data) < 8 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize SerializedType")
} }
val := binary.BigEndian.Uint64(value.Data[0:8])
value.Data = value.Data[8:]
i := reflect.ValueOf(val)
return i.Type(), &i, value, nil err = ctx.RegisterKind(reflect.Uint32, UInt32Type, SerializeUintN(4), DeserializeUintN[uint32](4))
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
err = ctx.RegisterKind(reflect.Int64, Int64Type, err = ctx.RegisterKind(reflect.Uint64, UInt64Type, SerializeUintN(8), DeserializeUintN[uint64](8))
func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ if err != nil {
var data []byte = nil return nil, err
if value != nil {
data = make([]byte, 8)
binary.BigEndian.PutUint64(data, uint64(value.Int()))
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
if value.Data == nil {
return reflect.TypeOf(int64(0)), nil, value, nil
} else {
if len(value.Data) < 8 {
return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize SerializedType")
} }
val := int64(binary.BigEndian.Uint64(value.Data[0:8]))
value.Data = value.Data[8:]
i := reflect.ValueOf(val)
return i.Type(), &i, value, nil err = ctx.RegisterKind(reflect.Uint, UIntType, SerializeUintN(8), DeserializeUintN[uint](8))
}
})
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -1195,10 +933,50 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) {
return nil, err return nil, err
} }
// TODO: move functions for string serialize/deserialize out of RegisterKind
/*
err = ctx.RegisterType(reflect.TypeOf(StringError("")), ErrorType, nil, nil) err = ctx.RegisterType(reflect.TypeOf(StringError("")), ErrorType, nil, nil)
if err != nil { if err != nil {
return nil, err return nil, err
} }
*/
err = ctx.RegisterType(reflect.TypeOf(ExtType(0)), ExtTypeType, SerializeUintN(4), DeserializeUintN[ExtType](4))
if err != nil {
return nil, err
}
// TODO: Make registering interfaces cleaner
var extension Extension = nil
err = ctx.RegisterType(reflect.ValueOf(&extension).Type().Elem(), ExtensionType, SerializeInterface, DeserializeInterface[Extension]())
if err != nil {
return nil, err
}
err = ctx.RegisterType(reflect.TypeOf(ListenerExt{}), SerializedType(ListenerExtType), SerializeStruct[ListenerExt](ctx), DeserializeStruct[ListenerExt](ctx))
if err != nil {
return nil, err
}
err = ctx.RegisterType(reflect.TypeOf(GroupExt{}), SerializedType(GroupExtType), SerializeStruct[GroupExt](ctx), DeserializeStruct[GroupExt](ctx))
if err != nil {
return nil, err
}
err = ctx.RegisterType(reflect.TypeOf(GQLExt{}), SerializedType(GQLExtType), SerializeStruct[GQLExt](ctx), DeserializeStruct[GQLExt](ctx))
if err != nil {
return nil, err
}
err = ctx.RegisterType(reflect.TypeOf(NodeType(0)), NodeTypeType, SerializeUintN(4), DeserializeUintN[NodeType](4))
if err != nil {
return nil, err
}
err = ctx.RegisterType(reflect.TypeOf(Node{}), NodeStructType, SerializeStruct[Node](ctx), DeserializeStruct[Node](ctx))
if err != nil {
return nil, err
}
err = ctx.RegisterType(reflect.TypeOf(RandID()), NodeIDType, err = ctx.RegisterType(reflect.TypeOf(RandID()), NodeIDType,
func(ctx *Context, ctx_type SerializedType, t reflect.Type, value *reflect.Value) (SerializedValue, error) { func(ctx *Context, ctx_type SerializedType, t reflect.Type, value *reflect.Value) (SerializedValue, error) {

14
node.go
Unescape Escape View File

@ -88,25 +88,25 @@ type PendingSignal struct {
// Default message channel size for nodes // Default message channel size for nodes
// Nodes represent a group of extensions that can be collectively addressed // Nodes represent a group of extensions that can be collectively addressed
type Node struct { type Node struct {
Key ed25519.PrivateKey `gv:""` Key ed25519.PrivateKey `gv:"key"`
ID NodeID ID NodeID
Type NodeType `gv:""` Type NodeType `gv:"type"`
Extensions map[ExtType]Extension `gv:"extensions"` Extensions map[ExtType]Extension `gv:"extensions"`
Policies map[PolicyType]Policy `gv:""` Policies map[PolicyType]Policy `gv:"policies"`
PendingACLs map[uuid.UUID]PendingACL `gv:""` PendingACLs map[uuid.UUID]PendingACL `gv:"pending_acls"`
PendingSignals map[uuid.UUID]PendingSignal `gv:""` PendingSignals map[uuid.UUID]PendingSignal `gv:"pending_signal"`
// Channel for this node to receive messages from the Context // Channel for this node to receive messages from the Context
MsgChan chan *Message MsgChan chan *Message
// Size of MsgChan // Size of MsgChan
BufferSize uint32 `gv:""` BufferSize uint32 `gv:"buffer_size"`
// Channel for this node to process delayed signals // Channel for this node to process delayed signals
TimeoutChan <-chan time.Time TimeoutChan <-chan time.Time
Active atomic.Bool Active atomic.Bool
SignalQueue []QueuedSignal `gv:""` SignalQueue []QueuedSignal `gv:"signal_queue"`
NextSignal *QueuedSignal NextSignal *QueuedSignal
} }

352
serialize.go
Unescape Escape View File

@ -5,6 +5,7 @@ import (
"encoding/binary" "encoding/binary"
"fmt" "fmt"
"reflect" "reflect"
"sort"
) )
const ( const (
@ -25,6 +26,11 @@ func Hash(base string, name string) SerializedType {
} }
type SerializedType uint64 type SerializedType uint64
func (t SerializedType) String() string {
return fmt.Sprintf("0x%x", uint64(t))
}
type ExtType SerializedType type ExtType SerializedType
type NodeType SerializedType type NodeType SerializedType
type SignalType SerializedType type SignalType SerializedType
@ -102,9 +108,353 @@ var (
ReqStateType = NewSerializedType("REQ_STATE") ReqStateType = NewSerializedType("REQ_STATE")
SignalDirectionType = NewSerializedType("SIGNAL_DIRECTION") SignalDirectionType = NewSerializedType("SIGNAL_DIRECTION")
NodeStructType = NewSerializedType("NODE_STRUCT")
NodeTypeType = NewSerializedType("NODE_TYPE")
ExtTypeType = NewSerializedType("EXT_TYPE")
ExtensionType = NewSerializedType("EXTENSION")
NodeIDType = NewSerializedType("NODE_ID") NodeIDType = NewSerializedType("NODE_ID")
) )
func SerializeUintN(size int)(func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue,error)){
var fill_data func([]byte, uint64) = nil
switch size {
case 1:
fill_data = func(data []byte, val uint64) {
data[0] = byte(val)
}
case 2:
fill_data = func(data []byte, val uint64) {
binary.BigEndian.PutUint16(data, uint16(val))
}
case 4:
fill_data = func(data []byte, val uint64) {
binary.BigEndian.PutUint32(data, uint32(val))
}
case 8:
fill_data = func(data []byte, val uint64) {
binary.BigEndian.PutUint64(data, val)
}
default:
panic(fmt.Sprintf("Cannot serialize uint of size %d", size))
}
return func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue,error){
var data []byte = nil
if value != nil {
data = make([]byte, size)
fill_data(data, value.Uint())
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}
}
func DeserializeUintN[T interface{~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64}](size int)(func(ctx *Context, value SerializedValue)(reflect.Type,*reflect.Value,SerializedValue,error)){
var get_uint func([]byte) uint64
switch size {
case 1:
get_uint = func(data []byte) uint64 {
return uint64(data[0])
}
case 2:
get_uint = func(data []byte) uint64 {
return uint64(binary.BigEndian.Uint16(data))
}
case 4:
get_uint = func(data []byte) uint64 {
return uint64(binary.BigEndian.Uint32(data))
}
case 8:
get_uint = func(data []byte) uint64 {
return binary.BigEndian.Uint64(data)
}
default:
panic(fmt.Sprintf("Cannot deserialize int of size %d", size))
}
var zero T
uint_type := reflect.TypeOf(zero)
return func(ctx *Context, value SerializedValue)(reflect.Type,*reflect.Value,SerializedValue,error){
if value.Data == nil {
return uint_type, nil, value, nil
} else {
var uint_bytes []byte
var err error
uint_bytes, value, err = value.PopData(size)
if err != nil {
return nil, nil, value, err
}
uint_value := reflect.New(uint_type).Elem()
uint_value.SetUint(get_uint(uint_bytes))
return uint_type, &uint_value, value, nil
}
}
}
func SerializeIntN(size int)(func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue,error)){
var fill_data func([]byte, int64) = nil
switch size {
case 1:
fill_data = func(data []byte, val int64) {
data[0] = byte(val)
}
case 2:
fill_data = func(data []byte, val int64) {
binary.BigEndian.PutUint16(data, uint16(val))
}
case 4:
fill_data = func(data []byte, val int64) {
binary.BigEndian.PutUint32(data, uint32(val))
}
case 8:
fill_data = func(data []byte, val int64) {
binary.BigEndian.PutUint64(data, uint64(val))
}
default:
panic(fmt.Sprintf("Cannot serialize int of size %d", size))
}
return func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue,error){
var data []byte = nil
if value != nil {
data = make([]byte, size)
fill_data(data, value.Int())
}
return SerializedValue{
[]SerializedType{ctx_type},
data,
}, nil
}
}
func DeserializeIntN[T interface{~int | ~int8 | ~int16 | ~int32 | ~int64}](size int)(func(ctx *Context, value SerializedValue)(reflect.Type,*reflect.Value,SerializedValue,error)){
var get_int func([]byte) int64
switch size {
case 1:
get_int = func(data []byte) int64 {
return int64(data[0])
}
case 2:
get_int = func(data []byte) int64 {
return int64(binary.BigEndian.Uint16(data))
}
case 4:
get_int = func(data []byte) int64 {
return int64(binary.BigEndian.Uint32(data))
}
case 8:
get_int = func(data []byte) int64 {
return int64(binary.BigEndian.Uint64(data))
}
default:
panic(fmt.Sprintf("Cannot deserialize int of size %d", size))
}
var zero T
int_type := reflect.TypeOf(zero)
return func(ctx *Context, value SerializedValue)(reflect.Type,*reflect.Value,SerializedValue,error){
if value.Data == nil {
return int_type, nil, value, nil
} else {
var int_bytes []byte
var err error
int_bytes, value, err = value.PopData(size)
if err != nil {
return nil, nil, value, err
}
int_value := reflect.New(int_type).Elem()
int_value.SetInt(get_int(int_bytes))
return int_type, &int_value, value, nil
}
}
}
type FieldInfo struct {
Index []int
TypeStack []SerializedType
}
type StructInfo struct {
Type reflect.Type
FieldOrder []SerializedType
FieldMap map[SerializedType]FieldInfo
}
func structInfo[T any](ctx *Context)StructInfo{
var struct_zero T
struct_type := reflect.TypeOf(struct_zero)
field_order := []SerializedType{}
field_map := map[SerializedType]FieldInfo{}
for _, field := range(reflect.VisibleFields(struct_type)) {
gv_tag, tagged_gv := field.Tag.Lookup("gv")
if tagged_gv == false {
continue
} else {
field_hash := Hash(FieldNameBase, gv_tag)
_, exists := field_map[field_hash]
if exists == true {
panic(fmt.Sprintf("gv tag %s is repeated", gv_tag))
} else {
field_serialized, err := SerializeValue(ctx, field.Type, nil)
if err != nil {
panic(err)
}
field_map[field_hash] = FieldInfo{
field.Index,
field_serialized.TypeStack,
}
field_order = append(field_order, field_hash)
}
}
}
sort.Slice(field_order, func(i, j int)bool {
return uint64(field_order[i]) < uint64(field_order[j])
})
return StructInfo{
struct_type,
field_order,
field_map,
}
}
func SerializeStruct[T any](ctx *Context)(func(*Context,SerializedType,reflect.Type,*reflect.Value)(SerializedValue,error)){
struct_info := structInfo[T](ctx)
return func(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue,error){
type_stack := []SerializedType{ctx_type}
var data []byte
if value == nil {
data = nil
} else {
data = make([]byte, 8)
for _, field_hash := range(struct_info.FieldOrder) {
field_hash_bytes := make([]byte, 8)
binary.BigEndian.PutUint64(field_hash_bytes, uint64(field_hash))
field_info := struct_info.FieldMap[field_hash]
field_value := value.FieldByIndex(field_info.Index)
field_serialized, err := SerializeValue(ctx, field_value.Type(), &field_value)
if err != nil {
return SerializedValue{}, err
}
data = append(data, field_hash_bytes...)
data = append(data, field_serialized.Data...)
}
binary.BigEndian.PutUint64(data[0:8], uint64(len(struct_info.FieldOrder)))
}
return SerializedValue{
type_stack,
data,
}, nil
}
}
func DeserializeStruct[T any](ctx *Context)(func(*Context,SerializedValue)(reflect.Type,*reflect.Value,SerializedValue,error)){
struct_info := structInfo[T](ctx)
return func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error) {
if value.Data == nil {
return struct_info.Type, nil, value, nil
} else {
var num_fields_bytes []byte
var err error
num_fields_bytes, value, err = value.PopData(8)
if err != nil {
return nil, nil, value, err
}
num_fields := int(binary.BigEndian.Uint64(num_fields_bytes))
struct_value := reflect.New(struct_info.Type).Elem()
for i := 0; i < num_fields; i += 1 {
var field_hash_bytes []byte
field_hash_bytes, value, err = value.PopData(8)
if err != nil {
return nil, nil, value, err
}
field_hash := SerializedType(binary.BigEndian.Uint64(field_hash_bytes))
field_info, exists := struct_info.FieldMap[field_hash]
if exists == false {
return nil, nil, value, fmt.Errorf("Field 0x%x is not valid for %+v: %d", field_hash, struct_info.Type, i)
}
field_value := struct_value.FieldByIndex(field_info.Index)
tmp_value := SerializedValue{
field_info.TypeStack,
value.Data,
}
var field_reflect *reflect.Value
_, field_reflect, tmp_value, err = DeserializeValue(ctx, tmp_value)
if err != nil {
return nil, nil, value, err
}
value.Data = tmp_value.Data
field_value.Set(*field_reflect)
}
return struct_info.Type, &struct_value, value, err
}
}
}
func SerializeInterface(ctx *Context, ctx_type SerializedType, reflect_type reflect.Type, value *reflect.Value)(SerializedValue,error){
var data []byte
type_stack := []SerializedType{ctx_type}
if value == nil {
data = nil
} else if value.IsZero() {
data = []byte{0x01}
} else {
data = []byte{0x00}
elem_value := value.Elem()
elem, err := SerializeValue(ctx, elem_value.Type(), &elem_value)
if err != nil {
return SerializedValue{}, err
}
elem_data, err := elem.MarshalBinary()
if err != nil {
return SerializedValue{}, err
}
data = append(data, elem_data...)
}
return SerializedValue{
type_stack,
data,
}, nil
}
func DeserializeInterface[T any]()(func(*Context,SerializedValue)(reflect.Type,*reflect.Value,SerializedValue,error)){
return func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){
var interface_zero T
var interface_type = reflect.ValueOf(&interface_zero).Type().Elem()
if value.Data == nil {
return interface_type, nil, value, nil
} else {
var flag_bytes []byte
var err error
flag_bytes, value, err = value.PopData(1)
if err != nil {
return nil, nil, value, err
}
interface_value := reflect.New(interface_type).Elem()
nil_flag := flag_bytes[0]
if nil_flag == 0x01 {
} else if nil_flag == 0x00 {
var elem_value *reflect.Value
var elem_ser SerializedValue
elem_ser, value.Data, err = ParseSerializedValue(value.Data)
_, elem_value, _, err = DeserializeValue(ctx, elem_ser)
if err != nil {
return nil, nil, value, err
}
interface_value.Set(*elem_value)
} else {
return nil, nil, value, fmt.Errorf("Unknown interface nil_flag value 0x%x", nil_flag)
}
return interface_type, &interface_value, value, nil
}
}
}
type SerializedValue struct { type SerializedValue struct {
TypeStack []SerializedType TypeStack []SerializedType
Data []byte Data []byte
@ -247,7 +597,7 @@ func DeserializeValue(ctx *Context, value SerializedValue) (reflect.Type, *refle
ctx_name = kind_info.Reflect.String() ctx_name = kind_info.Reflect.String()
} }
ctx.Log.Logf("serialize", "Deserializing: %+v(0x%d) - %+v", ctx_name, ctx_type, value.TypeStack) ctx.Log.Logf("serialize", "Deserializing: %+v(0x%d) - %+v", ctx_name, ctx_type, deserialize)
if value.Data == nil { if value.Data == nil {
reflect_type, _, value, err = deserialize(ctx, value) reflect_type, _, value, err = deserialize(ctx, value)

89
serialize_test.go
Unescape Escape View File

@ -29,6 +29,16 @@ func TestSerializeBasic(t *testing.T) {
testSerialize(t, ctx, map[int8]map[*int8]string{}) testSerialize(t, ctx, map[int8]map[*int8]string{})
var i interface{} = nil
testSerialize(t, ctx, i)
testSerializeMap(t, ctx, map[int8]interface{}{
0: "abcd",
1: uint32(12345678),
2: i,
3: 123,
})
testSerializeMap(t, ctx, map[int8]int32{ testSerializeMap(t, ctx, map[int8]int32{
0: 1234, 0: 1234,
2: 5678, 2: 5678,
@ -45,18 +55,63 @@ func TestSerializeBasic(t *testing.T) {
}) })
} }
type test struct {
Int int `gv:"int"`
Str string `gv:"string"`
}
func (s test) String() string {
return fmt.Sprintf("%d:%s", s.Int, s.Str)
}
func TestSerializeStructTags(t *testing.T) {
ctx := logTestContext(t, []string{"test", "serialize"})
test_type := NewSerializedType("TEST_STRUCT")
ctx.Log.Logf("test", "TEST_TYPE: %+v", test_type)
ctx.RegisterType(reflect.TypeOf(test{}), test_type, SerializeStruct[test](ctx), DeserializeStruct[test](ctx))
test_int := 10
test_string := "test"
ret := testSerialize(t, ctx, test{
test_int,
test_string,
})
if ret.Int != test_int {
t.Fatalf("Deserialized int %d does not equal test %d", ret.Int, test_int)
} else if ret.Str != test_string {
t.Fatalf("Deserialized string %s does not equal test %s", ret.Str, test_string)
}
testSerialize(t, ctx, []test{
{
test_int,
test_string,
},
{
test_int * 2,
fmt.Sprintf("%s%s", test_string, test_string),
},
{
test_int * 4,
fmt.Sprintf("%s%s%s", test_string, test_string, test_string),
},
})
}
func testSerializeMap[M map[T]R, T, R comparable](t *testing.T, ctx *Context, val M) { func testSerializeMap[M map[T]R, T, R comparable](t *testing.T, ctx *Context, val M) {
v := testSerialize(t, ctx, val) v := testSerialize(t, ctx, val)
for key, value := range(val) { for key, value := range(val) {
recreated, exists := v[key] recreated, exists := v[key]
if exists == false { if exists == false {
t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) t.Fatalf("DeserializeValue returned wrong value %+v != %+v", v, val)
} else if recreated != value { } else if recreated != value {
t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) t.Fatalf("DeserializeValue returned wrong value %+v != %+v", v, val)
} }
} }
if len(v) != len(val) { if len(v) != len(val) {
t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) t.Fatalf("DeserializeValue returned wrong value %+v != %+v", v, val)
} }
} }
@ -64,11 +119,11 @@ func testSerializeSliceSlice[S [][]T, T comparable](t *testing.T, ctx *Context,
v := testSerialize(t, ctx, val) v := testSerialize(t, ctx, val)
for i, original := range(val) { for i, original := range(val) {
if (original == nil && v[i] != nil) || (original != nil && v[i] == nil) { if (original == nil && v[i] != nil) || (original != nil && v[i] == nil) {
t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) t.Fatalf("DeserializeValue returned wrong value %+v != %+v", v, val)
} }
for j, o := range(original) { for j, o := range(original) {
if v[i][j] != o { if v[i][j] != o {
t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) t.Fatalf("DeserializeValue returned wrong value %+v != %+v", v, val)
} }
} }
} }
@ -149,11 +204,12 @@ func testSerializeStruct[T any](t *testing.T, ctx *Context, val T) {
} }
func testSerialize[T any](t *testing.T, ctx *Context, val T) T { func testSerialize[T any](t *testing.T, ctx *Context, val T) T {
value, err := SerializeAny(ctx, val) value := reflect.ValueOf(&val).Elem()
value_serialized, err := SerializeValue(ctx, value.Type(), &value)
fatalErr(t, err) fatalErr(t, err)
ctx.Log.Logf("test", "Serialized %+v to %+v", val, value) ctx.Log.Logf("test", "Serialized %+v to %+v", val, value_serialized)
ser, err := value.MarshalBinary() ser, err := value_serialized.MarshalBinary()
fatalErr(t, err) fatalErr(t, err)
ctx.Log.Logf("test", "Binary: %+v", ser) ctx.Log.Logf("test", "Binary: %+v", ser)
@ -165,21 +221,26 @@ func testSerialize[T any](t *testing.T, ctx *Context, val T) T {
t.Fatal("Data remaining after deserializing value") t.Fatal("Data remaining after deserializing value")
} }
val_type, deserialized_values, remaining_deserialize, err := DeserializeValue(ctx, val_parsed) val_type, deserialized_value, remaining_deserialize, err := DeserializeValue(ctx, val_parsed)
fatalErr(t, err) fatalErr(t, err)
if len(remaining_deserialize.Data) != 0 { if len(remaining_deserialize.Data) != 0 {
t.Fatal("Data remaining after deserializing value") t.Fatal("Data remaining after deserializing value")
} else if len(remaining_deserialize.TypeStack) != 0 { } else if len(remaining_deserialize.TypeStack) != 0 {
t.Fatal("TypeStack remaining after deserializing value") t.Fatal("TypeStack remaining after deserializing value")
} else if val_type != reflect.TypeOf(val) { } else if val_type != value.Type() {
t.Fatal(fmt.Sprintf("DeserializeValue returned wrong reflect.Type %+v - %+v", val_type, reflect.TypeOf(val))) t.Fatal(fmt.Sprintf("DeserializeValue returned wrong reflect.Type %+v - %+v", val_type, reflect.TypeOf(val)))
} else if deserialized_values == nil { } else if deserialized_value == nil {
t.Fatal("DeserializeValue returned no []reflect.Value") t.Fatal("DeserializeValue returned no []reflect.Value")
} else if deserialized_values == nil { } else if deserialized_value == nil {
t.Fatal("DeserializeValue returned nil *reflect.Value") t.Fatal("DeserializeValue returned nil *reflect.Value")
} else if deserialized_values.CanConvert(val_type) == false { } else if deserialized_value.CanConvert(val_type) == false {
t.Fatal("DeserializeValue returned value that can't convert to original value") t.Fatal("DeserializeValue returned value that can't convert to original value")
} }
return deserialized_values.Interface().(T) ctx.Log.Logf("test", "Value: %+v", deserialized_value.Interface())
if val_type.Kind() == reflect.Interface && deserialized_value.Interface() == nil {
var zero T
return zero
}
return deserialized_value.Interface().(T)
} }