diff --git a/context.go b/context.go index 3399e54..776cfa5 100644 --- a/context.go +++ b/context.go @@ -6,6 +6,7 @@ import ( "encoding/binary" "errors" "fmt" + "math" "reflect" "runtime" "sync" @@ -100,7 +101,7 @@ type NodeInfo struct { } type TypeSerialize func(*Context,uint64,reflect.Type,*reflect.Value) (SerializedValue, error) -type TypeDeserialize func(*Context,[]uint64,[]byte) (reflect.Type, *reflect.Value, []byte, error) +type TypeDeserialize func(*Context,SerializedValue) (reflect.Type, *reflect.Value, SerializedValue, error) type TypeInfo struct { Type reflect.Type Serialize TypeSerialize @@ -379,9 +380,9 @@ func (value SerializedValue) SerializedSize() uint64 { return uint64((len(value.TypeStack) + 2) * 8) } -func ParseSerializedValue(ctx *Context, data []byte) (SerializedValue, error) { +func ParseSerializedValue(data []byte) (SerializedValue, []byte, error) { if len(data) < 8 { - return SerializedValue{}, fmt.Errorf("SerializedValue required to have at least 8 bytes when serialized") + return SerializedValue{}, nil, fmt.Errorf("SerializedValue required to have at least 8 bytes when serialized") } num_types := int(binary.BigEndian.Uint64(data[0:8])) data_size := int(binary.BigEndian.Uint64(data[8:16])) @@ -393,20 +394,22 @@ func ParseSerializedValue(ctx *Context, data []byte) (SerializedValue, error) { } types_end := 8*(num_types + 2) + data_end := types_end + data_size return SerializedValue{ type_stack, - data[types_end:(types_end+data_size)], - }, nil + data[types_end:data_end], + }, data[data_end:], nil } -func DeserializeValue(ctx *Context, type_stack []uint64, data []byte, n int) (reflect.Type, []reflect.Value, []byte, error) { +func DeserializeValue(ctx *Context, value SerializedValue, n int) (reflect.Type, []reflect.Value, SerializedValue, error) { + ctx.Log.Logf("serialize", "DeserializeValue: %+v - %d", value, n) ret := make([]reflect.Value, n) var deserialize TypeDeserialize = nil var reflect_type reflect.Type = nil - ctx_type := type_stack[0] - type_stack = type_stack[1:] + ctx_type := value.TypeStack[0] + value.TypeStack = value.TypeStack[1:] type_info, exists := ctx.Types[SerializedType(ctx_type)] if exists == true { @@ -415,41 +418,38 @@ func DeserializeValue(ctx *Context, type_stack []uint64, data []byte, n int) (re } else { kind, exists := ctx.KindTypes[SerializedType(ctx_type)] if exists == false { - return nil, nil, nil, fmt.Errorf("Cannot deserialize 0x%x: unknown type/kind", ctx_type) + return nil, nil, SerializedValue{}, fmt.Errorf("Cannot deserialize 0x%x: unknown type/kind", ctx_type) } kind_info := ctx.Kinds[kind] deserialize = kind_info.Deserialize } - remaining_data := data - var value_type reflect.Type = nil - var err error = nil - if data == nil { - reflect_type, _, _, err = deserialize(ctx, type_stack, nil) + ctx.Log.Logf("serialize", "Deserializing: %d x %d", ctx_type, n) + + if value.Data == nil { + reflect_type, _, val, err := deserialize(ctx, value) if err != nil { - return nil, nil, nil, err + return nil, nil, SerializedValue{}, err } - } else { - for i := 0; i < n; i += 1 { - var elem *reflect.Value - var elem_type reflect.Type = nil - elem_type, elem, remaining_data, err = deserialize(ctx, type_stack, remaining_data) - if err != nil { - return nil, nil, nil, err - } - if value_type == nil { - value_type = elem_type - } - if len(remaining_data) == 0 { - remaining_data = nil - } - if elem == nil { - return nil, nil, nil, fmt.Errorf("root deserialize returned no value") - } - ret[i] = *elem + return reflect_type, nil, val, nil + } + + val := SerializedValue{ + value.TypeStack, + value.Data, + } + for i := 0; i < n; i += 1 { + val.TypeStack = value.TypeStack + var elem *reflect.Value + var err error + reflect_type, elem, val, err = deserialize(ctx, val) + if err != nil { + return nil, nil, SerializedValue{}, err } + ret[i] = *elem } - return reflect_type, ret, remaining_data, nil + ctx.Log.Logf("serialize", "DeserializeValue: DONE %+v - %+v", val, ret) + return reflect_type, ret, val, nil } // Create a new Context with the base library content added @@ -496,38 +496,36 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { append([]uint64{ctx_type}, elem.TypeStack...), data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte) (reflect.Type, *reflect.Value, []byte, error) { - // TODO: figure out how to deal with the case where the pointer value is nil - // In that case need to figure out the type of the pointer to create the nil pointer, which involves continuing to recuse the type stack without parsing any data - if data == nil { - elem_type, _, _, err := DeserializeValue(ctx, type_stack, nil, 1) + }, func(ctx *Context, value SerializedValue) (reflect.Type, *reflect.Value, SerializedValue, error) { + if value.Data == nil { + elem_type, _, _, err := DeserializeValue(ctx, value, 1) if err != nil { - return nil, nil, nil, err + return nil, nil, SerializedValue{}, err } - return elem_type, nil, nil, nil + return reflect.PointerTo(elem_type), nil, value, nil + } else if len(value.Data) < 1 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize pointer") } else { - if len(data) < 1 { - return nil, nil, nil, fmt.Errorf("Not enough data to deserialize pointer") - } - pointer_flags := data[0] + pointer_flags := value.Data[0] + value.Data = value.Data[1:] if pointer_flags == 0x00 { - _, elem_value, remaining_data, err := DeserializeValue(ctx, type_stack, data[1:], 1) + _, elem_value, remaining_data, err := DeserializeValue(ctx, value, 1) if err != nil { - return nil, nil, nil, err + return nil, nil, SerializedValue{}, err } pointer_value := elem_value[0].Addr() return pointer_value.Type(), &pointer_value, remaining_data, nil } else if pointer_flags == 0x01 { - elem_type, _, _, err := DeserializeValue(ctx, type_stack, nil, 1) + elem_type, _, remaining_data, err := DeserializeValue(ctx, value, 1) if err != nil { - return nil, nil, nil, err + return nil, nil, SerializedValue{}, err } pointer_type := reflect.PointerTo(elem_type) pointer_value := reflect.New(pointer_type).Elem() - return pointer_type, &pointer_value, data[1:], nil + return pointer_type, &pointer_value, remaining_data, nil } else { - return nil, nil, nil, fmt.Errorf("unknown pointer flags: %d", pointer_flags) + return nil, nil, SerializedValue{}, fmt.Errorf("unknown pointer flags: %d", pointer_flags) } } }) @@ -537,82 +535,188 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { err = ctx.RegisterKind(reflect.Struct, NewSerializedType("struct"), func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ - var m map[int][]byte = nil - if value != nil { - m = map[int][]byte{} + // TODO: Switch from serializing each field as a []byte of a SerializedValue.MarshalBinary to just running serializeValue and adding the TypeStack and Data together + serialized_value := SerializedValue{ + []uint64{ctx_type}, + nil, } num_fields := 0 + field_values := map[int]SerializedValue{} for _, field := range(reflect.VisibleFields(reflect_type)) { gv_tag, tagged_gv := field.Tag.Lookup("gv") if tagged_gv == false { continue } else if gv_tag == "" { continue - } else if m != nil { + } else { + // Add to the type stack and data stack field_index, err := strconv.Atoi(gv_tag) if err != nil { return SerializedValue{}, err } num_fields += 1 - - field_value := value.FieldByIndex(field.Index) - field_ser, err := serializeValue(ctx, field.Type, &field_value) - if err != nil { - return SerializedValue{}, err - } - - m[field_index], err = field_ser.MarshalBinary() - if err != nil { - return SerializedValue{}, nil + if value == nil { + field_ser, err := serializeValue(ctx, field.Type, nil) + if err != nil { + return SerializedValue{}, err + } + field_values[field_index] = field_ser + } else { + field_value := value.FieldByIndex(field.Index) + field_ser, err := serializeValue(ctx, field.Type, &field_value) + if err != nil { + return SerializedValue{}, err + } + field_values[field_index] = field_ser } } } - field_list := make([][]byte, num_fields) - for i := range(field_list) { - var exists bool = false - field_list[i], exists = m[i] + + for i := 0; i < num_fields; i += 1 { + field_value, exists := field_values[i] if exists == false { return SerializedValue{}, fmt.Errorf("%+v missing gv:%d", reflect_type, i) } + serialized_value.TypeStack = append(serialized_value.TypeStack, field_value.TypeStack...) + if value != nil { + serialized_value.Data = append(serialized_value.Data, field_value.Data...) + } } - list_value := reflect.ValueOf(field_list) - list_serial, err := serializeValue(ctx, list_value.Type(), &list_value) - if err != nil { - return SerializedValue{}, err + return serialized_value, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return nil, nil, SerializedValue{}, fmt.Errorf("deserialize struct not implemented") + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Int, NewSerializedType("int"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = make([]byte, 8) + binary.BigEndian.PutUint64(data, uint64(value.Int())) } return SerializedValue{ []uint64{ctx_type}, - list_serial.Data, + data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize struct not implemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(0), nil, value, nil + } + if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("invalid length: %d/8", len(value.Data)) + } + int_val := reflect.ValueOf(int(binary.BigEndian.Uint64(value.Data[0:8]))) + value.Data = value.Data[8:] + return int_val.Type(), &int_val, value, nil }) if err != nil { return nil, err } - err = ctx.RegisterKind(reflect.Int, NewSerializedType("int"), + err = ctx.RegisterKind(reflect.Bool, NewSerializedType("bool"), func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ var data []byte = nil if value != nil { + b := value.Bool() + if b == true { + data = []byte{0x01} + } else { + data = []byte{0x00} + } + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(true), nil, value, nil + } else if len(value.Data) == 0 { + return nil, nil, SerializedValue{}, fmt.Errorf("not enough data to deserialize bool") + } else { + b := value.Data[0] + value.Data = value.Data[1:] + var val reflect.Value + switch b { + case 0x00: + val = reflect.ValueOf(false) + case 0x01: + val = reflect.ValueOf(true) + default: + return nil, nil, SerializedValue{}, fmt.Errorf("unknown boolean 0x%x", b) + } + return reflect.TypeOf(true), &val, value, nil + } + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Float64, NewSerializedType("float64"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + // TODO: fix if underlying memory layout of float32 changes(or if it's architecture-dependent) data = make([]byte, 8) - binary.BigEndian.PutUint64(data, value.Uint()) + val := math.Float64bits(float64(value.Float())) + binary.BigEndian.PutUint64(data, val) } return SerializedValue{ []uint64{ctx_type}, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - if len(data) < 8 { - return nil, nil, nil, fmt.Errorf("invalid length: %d/8", len(data)) + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(float64(0)), nil, value, nil + } else { + if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize float32") + } + val_int := binary.BigEndian.Uint64(value.Data[0:8]) + value.Data = value.Data[8:] + val := math.Float64frombits(val_int) + + float_val := reflect.ValueOf(val) + + return float_val.Type(), &float_val, value, nil } - remaining_data := data[8:] - if len(remaining_data) == 0 { - remaining_data = nil + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Float32, NewSerializedType("float32"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + // TODO: fix if underlying memory layout of float32 changes(or if it's architecture-dependent) + data = make([]byte, 4) + val := math.Float32bits(float32(value.Float())) + binary.BigEndian.PutUint32(data, val) + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(float32(0)), nil, value, nil + } else { + if len(value.Data) < 4 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize float32") + } + val_int := binary.BigEndian.Uint32(value.Data[0:4]) + value.Data = value.Data[4:] + val := math.Float32frombits(val_int) + + float_value := reflect.ValueOf(val) + + return float_value.Type(), &float_value, value, nil } - int_val := reflect.ValueOf(binary.BigEndian.Uint64(data[0:8])) - return int_val.Type(), &int_val, remaining_data, nil }) if err != nil { return nil, err @@ -628,8 +732,19 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{ctx_type}, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize uint32 unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(uint32(0)), nil, value, nil + } else { + if len(value.Data) < 4 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint32") + } + val := binary.BigEndian.Uint32(value.Data[0:4]) + value.Data = value.Data[4:] + int_value := reflect.ValueOf(val) + + return int_value.Type(), &int_value, value, nil + } }) if err != nil { return nil, err @@ -651,8 +766,22 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{uint64(ctx_type)}, append(data, []byte(str)...), }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize string unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(""), nil, value, nil + } else if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize string") + } else { + str_len := binary.BigEndian.Uint64(value.Data[0:8]) + value.Data = value.Data[8:] + if len(value.Data) < int(str_len) { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize string of length %d(%d)", str_len, len(value.Data)) + } + string_bytes := value.Data[:str_len] + value.Data = value.Data[str_len:] + str_value := reflect.ValueOf(string(string_bytes)) + return reflect.TypeOf(""), &str_value, value, nil + } }) if err != nil { return nil, err @@ -693,8 +822,8 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { append([]uint64{ctx_type}, elem.TypeStack...), data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize array unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return nil, nil, SerializedValue{}, fmt.Errorf("deserialize array unimplemented") }) if err != nil { return nil, err @@ -721,8 +850,8 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { append([]uint64{ctx_type}, type_stack...), data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize interface unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return nil, nil, SerializedValue{}, fmt.Errorf("deserialize interface unimplemented") }) if err != nil { return nil, err @@ -796,13 +925,122 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { type_stack, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize map unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + var key_type, elem_type reflect.Type + key_type, _, value, err = DeserializeValue(ctx, value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + elem_type, _, value, err = DeserializeValue(ctx, value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + return reflect.MapOf(key_type, elem_type), nil, value, nil + } else if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize map") + } else { + map_len := binary.BigEndian.Uint64(value.Data[0:8]) + value.Data = value.Data[8:] + if map_len == 0xFFFFFFFFFFFFFFFF { + temp_value := SerializedValue{ + value.TypeStack, + nil, + } + + var key_type, elem_type reflect.Type + key_type, _, temp_value, err = DeserializeValue(ctx, temp_value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + elem_type, _, temp_value, err = DeserializeValue(ctx, temp_value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + + map_type := reflect.MapOf(key_type, elem_type) + map_value := reflect.New(map_type).Elem() + + return map_type, &map_value, SerializedValue{ + temp_value.TypeStack, + value.Data, + }, nil + } else if map_len == 0x00 { + temp_value := SerializedValue{ + value.TypeStack, + nil, + } + + var key_type, elem_type reflect.Type + key_type, _, temp_value, err = DeserializeValue(ctx, temp_value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + elem_type, _, temp_value, err = DeserializeValue(ctx, temp_value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + + map_type := reflect.MapOf(key_type, elem_type) + map_value := reflect.MakeMap(map_type) + + return map_type, &map_value, SerializedValue{ + temp_value.TypeStack, + value.Data, + }, nil + } else { + var key_type, elem_type reflect.Type + var key_values, elem_values []reflect.Value + key_type, key_values, value, err = DeserializeValue(ctx, value, int(map_len)) + if err != nil { + return nil, nil, SerializedValue{}, err + } + elem_type, elem_values, value, err = DeserializeValue(ctx, value, int(map_len)) + if err != nil { + return nil, nil, SerializedValue{}, err + } + + map_type := reflect.MapOf(key_type, elem_type) + map_value := reflect.MakeMap(map_type) + + for i := 0; i < int(map_len); i += 1 { + map_value.SetMapIndex(key_values[i], elem_values[i]) + } + + return map_type, &map_value, value, nil + } + } }) if err != nil { return nil, err } + err = ctx.RegisterKind(reflect.Int8, NewSerializedType("int8"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = []byte{byte(value.Int())} + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(int8(0)), nil, value, nil + } else { + if len(value.Data) < 1 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize int8") + } + i := int8(value.Data[0]) + value.Data = value.Data[1:] + val := reflect.ValueOf(i) + return val.Type(), &val, value, nil + } + }) + if err != nil { + return nil, err + } err = ctx.RegisterKind(reflect.Uint8, NewSerializedType("uint8"), func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ @@ -814,8 +1052,134 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{ctx_type}, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize uint8 unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(uint8(0)), nil, value, nil + } else { + if len(value.Data) < 1 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint8") + } + i := uint8(value.Data[0]) + value.Data = value.Data[1:] + val := reflect.ValueOf(i) + return val.Type(), &val, value, nil + } + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Uint16, NewSerializedType("uint16"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = make([]byte, 2) + binary.BigEndian.PutUint16(data, uint16(value.Uint())) + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(uint16(0)), nil, value, nil + } else { + if len(value.Data) < 2 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint16") + } + val := binary.BigEndian.Uint16(value.Data[0:2]) + value.Data = value.Data[2:] + i := reflect.ValueOf(val) + + return i.Type(), &i, value, nil + } + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Int16, NewSerializedType("int16"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = make([]byte, 2) + binary.BigEndian.PutUint16(data, uint16(value.Int())) + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(int16(0)), nil, value, nil + } else { + if len(value.Data) < 2 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint16") + } + val := int16(binary.BigEndian.Uint16(value.Data[0:2])) + value.Data = value.Data[2:] + i := reflect.ValueOf(val) + + return i.Type(), &i, value, nil + } + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Int32, NewSerializedType("int32"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = make([]byte, 4) + binary.BigEndian.PutUint32(data, uint32(value.Int())) + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(int32(0)), nil, value, nil + } else { + if len(value.Data) < 4 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint16") + } + val := int32(binary.BigEndian.Uint32(value.Data[0:4])) + value.Data = value.Data[4:] + i := reflect.ValueOf(val) + + return i.Type(), &i, value, nil + } + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Uint, NewSerializedType("uint"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = make([]byte, 8) + binary.BigEndian.PutUint64(data, value.Uint()) + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(uint(0)), nil, value, nil + } else { + if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint64") + } + val := uint(binary.BigEndian.Uint64(value.Data[0:8])) + value.Data = value.Data[8:] + i := reflect.ValueOf(val) + + return i.Type(), &i, value, nil + } }) if err != nil { return nil, err @@ -832,8 +1196,48 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{ctx_type}, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("deserialize uint64 unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(uint64(0)), nil, value, nil + } else { + if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint64") + } + val := binary.BigEndian.Uint64(value.Data[0:8]) + value.Data = value.Data[8:] + i := reflect.ValueOf(val) + + return i.Type(), &i, value, nil + } + }) + if err != nil { + return nil, err + } + + err = ctx.RegisterKind(reflect.Int64, NewSerializedType("int64"), + func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ + var data []byte = nil + if value != nil { + data = make([]byte, 8) + binary.BigEndian.PutUint64(data, uint64(value.Int())) + } + return SerializedValue{ + []uint64{ctx_type}, + data, + }, nil + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + return reflect.TypeOf(int64(0)), nil, value, nil + } else { + if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize uint64") + } + val := int64(binary.BigEndian.Uint64(value.Data[0:8])) + value.Data = value.Data[8:] + i := reflect.ValueOf(val) + + return i.Type(), &i, value, nil + } }) if err != nil { return nil, err @@ -842,6 +1246,7 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { err = ctx.RegisterKind(reflect.Slice, NewSerializedType("slice"), func(ctx *Context, ctx_type uint64, reflect_type reflect.Type, value *reflect.Value)(SerializedValue, error){ var data []byte + var type_stack []uint64 = nil if value == nil { data = nil } else if value.IsZero() { @@ -851,7 +1256,6 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { } else { data := make([]byte, 8) binary.BigEndian.PutUint64(data, uint64(value.Len())) - var type_stack []uint64 for i := 0; i < value.Len(); i += 1 { val := value.Index(i) element, err := serializeValue(ctx, reflect_type.Elem(), &val) @@ -864,20 +1268,66 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { data = append(data, element.Data...) } return SerializedValue{ - append([]uint64{ctx_type}, type_stack...), + type_stack, data, }, nil } - elem, err := serializeValue(ctx, reflect_type.Elem(), nil) + element, err := serializeValue(ctx, reflect_type.Elem(), nil) if err != nil { return SerializedValue{}, err } return SerializedValue{ - elem.TypeStack, + append([]uint64{ctx_type}, element.TypeStack...), data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("not implemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + if value.Data == nil { + elem_type, _, _, err := DeserializeValue(ctx, value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + return reflect.SliceOf(elem_type), nil, value, nil + } else if len(value.Data) < 8 { + return nil, nil, SerializedValue{}, fmt.Errorf("Not enough data to deserialize slice") + } else { + slice_length := binary.BigEndian.Uint64(value.Data[0:8]) + value.Data = value.Data[8:] + if slice_length == 0xFFFFFFFFFFFFFFFF { + elem_type, _, remaining, err := DeserializeValue(ctx, SerializedValue{ + value.TypeStack, + nil, + }, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + reflect_type := reflect.SliceOf(elem_type) + reflect_value := reflect.New(reflect_type).Elem() + return reflect_type, &reflect_value, SerializedValue{ + remaining.TypeStack, + value.Data, + }, nil + } else if slice_length == 0x00 { + elem_type, _, remaining, err := DeserializeValue(ctx, value, 1) + if err != nil { + return nil, nil, SerializedValue{}, err + } + reflect_value := reflect.MakeSlice(reflect.SliceOf(elem_type), 0, 0) + return reflect_value.Type(), &reflect_value, SerializedValue{ + remaining.TypeStack, + value.Data, + }, nil + } else { + elem_type, elements, remaining_data, err := DeserializeValue(ctx, value, int(slice_length)) + if err != nil { + return nil, nil, SerializedValue{}, err + } + reflect_value := reflect.MakeSlice(reflect.SliceOf(elem_type), 0, int(slice_length)) + for i := 0; i < int(slice_length); i += 1 { + reflect_value = reflect.Append(reflect_value, elements[i]) + } + return reflect_value.Type(), &reflect_value, remaining_data, nil + } + } }) if err != nil { return nil, err @@ -900,8 +1350,8 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{uint64(ctx_type)}, append(data, []byte(str)...), }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return nil, nil, SerializedValue{}, fmt.Errorf("unimplemented") }) err = ctx.RegisterType(reflect.TypeOf(RandID()), NewSerializedType("NodeID"), @@ -918,8 +1368,8 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{ctx_type}, id_ser, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return nil, nil, nil, fmt.Errorf("unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return nil, nil, SerializedValue{}, fmt.Errorf("unimplemented") }) if err != nil { return nil, err @@ -936,8 +1386,8 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{ctx_type}, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return reflect.TypeOf(Up), nil, nil, fmt.Errorf("unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return reflect.TypeOf(Up), nil, SerializedValue{}, fmt.Errorf("unimplemented") }) if err != nil { return nil, err @@ -954,8 +1404,8 @@ func NewContext(db * badger.DB, log Logger) (*Context, error) { []uint64{ctx_type}, data, }, nil - }, func(ctx *Context, type_stack []uint64, data []byte)(reflect.Type, *reflect.Value, []byte, error){ - return reflect.TypeOf(ReqState(0)), nil, nil, fmt.Errorf("unimplemented") + }, func(ctx *Context, value SerializedValue)(reflect.Type, *reflect.Value, SerializedValue, error){ + return reflect.TypeOf(ReqState(0)), nil, SerializedValue{}, fmt.Errorf("unimplemented") }) if err != nil { return nil, err diff --git a/node.go b/node.go index 5f88032..a04de6e 100644 --- a/node.go +++ b/node.go @@ -398,8 +398,6 @@ func nodeLoop(ctx *Context, node *Node) error { msgs = msgs.Add(ctx, node.ID, node.Key, NewReadResultSignal(sig.ID, node.ID, node.Type, result), source) msgs = msgs.Add(ctx, node.ID, node.Key, NewErrorSignal(sig.ID, "read_done"), source) ctx.Send(msgs) - default: - println(fmt.Sprintf("NOT_SPECIAL_SIGNAL: %+v", reflect.TypeOf(sig))) } node.Process(ctx, source, signal) @@ -661,17 +659,22 @@ func LoadNode(ctx * Context, id NodeID) (*Node, error) { return nil, err } - value, err := ParseSerializedValue(ctx, bytes) + value, remaining, err := ParseSerializedValue(bytes) if err != nil { return nil, err + } else if len(remaining) != 0 { + return nil, fmt.Errorf("%d bytes left after parsing node from DB", len(remaining)) } - _, node_val, remaining, err := DeserializeValue(ctx, value.TypeStack, value.Data, 1) + _, node_val, remaining_data, err := DeserializeValue(ctx, value, 1) if err != nil { return nil, err } - if remaining != nil { - return nil, fmt.Errorf("%d bytes left after desrializing *Node", len(remaining)) + if len(remaining_data.TypeStack) != 0 { + return nil, fmt.Errorf("%d entries left in typestack after deserializing *Node", len(remaining_data.TypeStack)) + } + if len(remaining_data.Data) != 0 { + return nil, fmt.Errorf("%d bytes left after desrializing *Node", len(remaining_data.Data)) } node, ok := node_val[0].Interface().(*Node) diff --git a/policy.go b/policy.go index bfd36b6..bf5642b 100644 --- a/policy.go +++ b/policy.go @@ -64,7 +64,7 @@ func (policy *RequirementOfPolicy) ContinueAllows(ctx *Context, current PendingA return Deny } - _, reqs_if, _, err := DeserializeValue(ctx, reqs_ser.TypeStack, reqs_ser.Data, 1) + _, reqs_if, _, err := DeserializeValue(ctx, reqs_ser, 1) if err != nil { return Deny } @@ -113,7 +113,7 @@ func (policy *MemberOfPolicy) ContinueAllows(ctx *Context, current PendingACL, s return Deny } - _, members_if, _, err := DeserializeValue(ctx, members_ser.TypeStack, members_ser.Data, 1) + _, members_if, _, err := DeserializeValue(ctx, members_ser, 1) if err != nil { return Deny } diff --git a/serialize_test.go b/serialize_test.go new file mode 100644 index 0000000..d63c5ab --- /dev/null +++ b/serialize_test.go @@ -0,0 +1,127 @@ +package graphvent + +import ( + "testing" + "reflect" + "fmt" +) + +func TestSerializeBasic(t *testing.T) { + ctx := logTestContext(t, []string{"test", "serialize"}) + testSerializeComparable[string](t, ctx, "test") + testSerializeComparable[bool](t, ctx, true) + testSerializeComparable[float32](t, ctx, 0.05) + testSerializeComparable[float64](t, ctx, 0.05) + testSerializeComparable[uint](t, ctx, uint(1234)) + testSerializeComparable[uint8] (t, ctx, uint8(123)) + testSerializeComparable[uint16](t, ctx, uint16(1234)) + testSerializeComparable[uint32](t, ctx, uint32(12345)) + testSerializeComparable[uint64](t, ctx, uint64(123456)) + testSerializeComparable[int](t, ctx, 1234) + testSerializeComparable[int8] (t, ctx, int8(-123)) + testSerializeComparable[int16](t, ctx, int16(-1234)) + testSerializeComparable[int32](t, ctx, int32(-12345)) + testSerializeComparable[int64](t, ctx, int64(-123456)) + testSerializeSlice[[]int](t, ctx, []int{123, 456, 789, 101112}) + testSerializeSlice[[]int](t, ctx, ([]int)(nil)) + testSerializeSliceSlice[[][]int](t, ctx, [][]int{{123, 456, 789, 101112}, {3253, 2341, 735, 212}, {123, 51}, nil}) + testSerializeSliceSlice[[][]string](t, ctx, [][]string{{"123", "456", "789", "101112"}, {"3253", "2341", "735", "212"}, {"123", "51"}, nil}) + + testSerializeMap(t, ctx, map[int]int{ + 1: 2, + 3: 4, + 5: 6, + 7: 8, + }) + + testSerialize(t, ctx, struct{ + int `gv:"0"` + string `gv:"1"` + }{ + 12345, + "test_string", + }) +} + +func testSerializeMap[M map[T]R, T, R comparable](t *testing.T, ctx *Context, val M) { + v := testSerialize(t, ctx, val) + for key, value := range(val) { + recreated, exists := v[key] + if exists == false { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } else if recreated != value { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } + } + if len(v) != len(val) { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } +} + +func testSerializeSliceSlice[S [][]T, T comparable](t *testing.T, ctx *Context, val S) { + v := testSerialize(t, ctx, val) + for i, original := range(val) { + if (original == nil && v[i] != nil) || (original != nil && v[i] == nil) { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } + for j, o := range(original) { + if v[i][j] != o { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } + } + } +} + +func testSerializeSlice[S []T, T comparable](t *testing.T, ctx *Context, val S) { + v := testSerialize(t, ctx, val) + for i, original := range(val) { + if v[i] != original { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } + } +} + +func testSerializeComparable[T comparable](t *testing.T, ctx *Context, val T) { + v := testSerialize(t, ctx, val) + if v != val { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong value %+v != %+v", v, val)) + } +} + +func testSerialize[T any](t *testing.T, ctx *Context, val T) T { + value, err := SerializeValue(ctx, reflect.ValueOf(val)) + fatalErr(t, err) + ctx.Log.Logf("test", "Serialized %+v to %+v", val, value) + + ser, err := value.MarshalBinary() + fatalErr(t, err) + ctx.Log.Logf("test", "Binary: %+v", ser) + + val_parsed, remaining_parse, err := ParseSerializedValue(ser) + fatalErr(t, err) + ctx.Log.Logf("test", "Parsed: %+v", val_parsed) + + if len(remaining_parse) != 0 { + t.Fatal("Data remaining after deserializing value") + } + + val_type, deserialized_values, remaining_deserialize, err := DeserializeValue(ctx, val_parsed, 1) + fatalErr(t, err) + + if len(remaining_deserialize.Data) != 0 { + t.Fatal("Data remaining after deserializing value") + } else if len(remaining_deserialize.TypeStack) != 0 { + t.Fatal("TypeStack remaining after deserializing value") + } else if val_type != reflect.TypeOf(val) { + t.Fatal(fmt.Sprintf("DeserializeValue returned wrong reflect.Type %+v - %+v", val_type, reflect.TypeOf(val))) + } else if deserialized_values == nil { + t.Fatal("DeserializeValue returned no []reflect.Value") + } else if len(deserialized_values) == 0 { + t.Fatal("DeserializeValue returned empty []reflect.Value") + } else if len(deserialized_values) > 1 { + t.Fatal("DeserializeValue returned wrong length []reflect.Value") + } else if deserialized_values[0].CanConvert(val_type) == false { + t.Fatal("DeserializeValue returned value that can't convert to original value") + } + return deserialized_values[0].Interface().(T) +}