Initial commit

main
noah metz 2024-01-06 19:56:56 -07:00
commit 6753c220af
6 changed files with 1084 additions and 0 deletions

@ -0,0 +1,25 @@
package main
import (
"git.metznet.ca/MetzNet/boolean"
"fmt"
"bufio"
"os"
"strings"
)
func main() {
if len(os.Args) < 2 {
return
}
stream := bufio.NewReader(strings.NewReader(os.Args[1]))
token_stream, err := boolean.Tokenize(stream)
if err != nil {
panic(err)
}
expr, err := boolean.ParseExpr(token_stream)
if err != nil {
panic(err)
}
fmt.Printf("Expr: %s\n", expr)
}

@ -0,0 +1,640 @@
package boolean
import (
"fmt"
"strconv"
"errors"
"io"
"math"
)
type Expr interface {
String() string
Value(symbols map[string]Expr) Expr
}
type NumericConstant struct {
value float64
}
func (val NumericConstant) String() string {
return fmt.Sprintf("%f", val.value)
}
func (val NumericConstant) Value(symbols map[string]Expr) Expr {
return val
}
type BooleanConstant struct {
value bool
}
func (val BooleanConstant) String() string {
return fmt.Sprintf("%t", val.value)
}
func (val BooleanConstant) Value(symbols map[string]Expr) Expr {
return val
}
func ToNumeric(expr Expr) (float64, error) {
numeric, ok := expr.(NumericConstant)
if ok == false {
boolean, ok := expr.(BooleanConstant)
if ok == false {
return 0.0, fmt.Errorf("expr is not constant, can not convert to numeric")
} else if boolean.value {
return 1.0, nil
} else {
return 0.0, nil
}
} else {
return numeric.value, nil
}
}
func ToBoolean(expr Expr) (bool, error) {
boolean, ok := expr.(BooleanConstant)
if ok == false {
numeric, ok := expr.(NumericConstant)
if ok == false {
return false, fmt.Errorf("expr is not constant, can not convert to numeric")
} else {
return numeric.value == 0.0, nil
}
} else {
return boolean.value, nil
}
}
type LogicalNot struct {
sub_expr Expr
}
func(val LogicalNot) String() string {
return fmt.Sprintf("(NOT %s)", val.sub_expr)
}
func(val LogicalNot) Value(symbols map[string]Expr) Expr {
sub_expr := val.sub_expr.Value(symbols)
boolean, err := ToBoolean(sub_expr)
if err != nil {
return LogicalNot{sub_expr}
}
if boolean {
return BooleanConstant{false}
} else {
return BooleanConstant{true}
}
}
type NumericNegate struct {
sub_expr Expr
}
func(val NumericNegate) String() string {
return fmt.Sprintf("(- %s)", val.sub_expr)
}
func(val NumericNegate) Value(symbols map[string]Expr) Expr {
sub_expr := val.sub_expr.Value(symbols)
numeric, err := ToNumeric(sub_expr)
if err != nil {
return NumericNegate{sub_expr}
}
return NumericConstant{-numeric}
}
type NumericAdd struct {
left Expr
right Expr
}
func (expr NumericAdd) String() string {
return fmt.Sprintf("(+ %s %s)", expr.left, expr.right)
}
func (expr NumericAdd) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericAdd{left, right}
} else if left_err == nil && right_err != nil {
return NumericAdd{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericAdd{left, NumericConstant{right_numeric}}
}
return NumericConstant{left_numeric + right_numeric}
}
type NumericSub struct {
left Expr
right Expr
}
func (expr NumericSub) String() string {
return fmt.Sprintf("(- %s %s)", expr.left, expr.right)
}
func (expr NumericSub) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericSub{left, right}
} else if left_err == nil && right_err != nil {
return NumericSub{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericSub{left, NumericConstant{right_numeric}}
}
return NumericConstant{left_numeric - right_numeric}
}
type ErrorExpr struct {
text string
}
func (expr ErrorExpr) String() string {
return fmt.Sprintf("(ERR %s)", expr.text)
}
func (expr ErrorExpr) Value(symbols map[string]Expr) Expr {
return expr
}
type NumericDiv struct {
left Expr
right Expr
}
func (expr NumericDiv) String() string {
return fmt.Sprintf("(/ %s %s)", expr.left, expr.right)
}
func (expr NumericDiv) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericDiv{left, right}
} else if left_err == nil && right_err != nil {
return NumericDiv{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericDiv{left, NumericConstant{right_numeric}}
}
if right_numeric == 0 {
return ErrorExpr{"Divide by zero"}
}
return NumericConstant{left_numeric / right_numeric}
}
type NumericMul struct {
left Expr
right Expr
}
func (expr NumericMul) String() string {
return fmt.Sprintf("(* %s %s)", expr.left, expr.right)
}
func (expr NumericMul) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericMul{left, right}
} else if left_err == nil && right_err != nil {
return NumericMul{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericMul{left, NumericConstant{right_numeric}}
}
return NumericConstant{left_numeric * right_numeric}
}
type NumericPow struct {
left Expr
right Expr
}
func (expr NumericPow) String() string {
return fmt.Sprintf("(** %s %s)", expr.left, expr.right)
}
func (expr NumericPow) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericPow{left, right}
} else if left_err == nil && right_err != nil {
return NumericPow{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericPow{left, NumericConstant{right_numeric}}
}
return NumericConstant{math.Pow(left_numeric, right_numeric)}
}
type NumericEQ struct {
left Expr
right Expr
}
func (expr NumericEQ) String() string {
return fmt.Sprintf("(== %s %s)", expr.left, expr.right)
}
func (expr NumericEQ) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericEQ{left, right}
} else if left_err == nil && right_err != nil {
return NumericEQ{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericEQ{left, NumericConstant{right_numeric}}
}
return BooleanConstant{left_numeric == right_numeric}
}
type NumericNEQ struct {
left Expr
right Expr
}
func (expr NumericNEQ) String() string {
return fmt.Sprintf("(!= %s %s)", expr.left, expr.right)
}
func (expr NumericNEQ) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericNEQ{left, right}
} else if left_err == nil && right_err != nil {
return NumericNEQ{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericNEQ{left, NumericConstant{right_numeric}}
}
return BooleanConstant{left_numeric != right_numeric}
}
type NumericGT struct {
left Expr
right Expr
}
func (expr NumericGT) String() string {
return fmt.Sprintf("(> %s %s)", expr.left, expr.right)
}
func (expr NumericGT) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericGT{left, right}
} else if left_err == nil && right_err != nil {
return NumericGT{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericGT{left, NumericConstant{right_numeric}}
}
return BooleanConstant{left_numeric > right_numeric}
}
type NumericLT struct {
left Expr
right Expr
}
func (expr NumericLT) String() string {
return fmt.Sprintf("(< %s %s)", expr.left, expr.right)
}
func (expr NumericLT) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericLT{left, right}
} else if left_err == nil && right_err != nil {
return NumericLT{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericLT{left, NumericConstant{right_numeric}}
}
return BooleanConstant{left_numeric < right_numeric}
}
type NumericGEQ struct {
left Expr
right Expr
}
func (expr NumericGEQ) String() string {
return fmt.Sprintf("(>= %s %s)", expr.left, expr.right)
}
func (expr NumericGEQ) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericGEQ{left, right}
} else if left_err == nil && right_err != nil {
return NumericGEQ{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericGEQ{left, NumericConstant{right_numeric}}
}
return BooleanConstant{left_numeric >= right_numeric}
}
type NumericLEQ struct {
left Expr
right Expr
}
func (expr NumericLEQ) String() string {
return fmt.Sprintf("(<= %s %s)", expr.left, expr.right)
}
func (expr NumericLEQ) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_numeric, left_err := ToNumeric(left)
right_numeric, right_err := ToNumeric(right)
if left_err != nil && right_err != nil {
return NumericLEQ{left, right}
} else if left_err == nil && right_err != nil {
return NumericLEQ{NumericConstant{left_numeric}, right}
} else if left_err != nil && right_err == nil {
return NumericLEQ{left, NumericConstant{right_numeric}}
}
return BooleanConstant{left_numeric <= right_numeric}
}
type SymbolExpr struct {
name string
}
func (expr SymbolExpr) String() string {
return expr.name
}
func (expr SymbolExpr) Value(symbols map[string]Expr) Expr {
if symbols == nil {
return expr
}
value, has_value := symbols[expr.name]
if has_value {
return value
} else {
return expr
}
}
type LogicalAnd struct {
left Expr
right Expr
}
func (expr LogicalAnd) String() string {
return fmt.Sprintf("(& %s %s)", expr.left, expr.right)
}
func (expr LogicalAnd) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_boolean, left_err := ToBoolean(left)
right_boolean, right_err := ToBoolean(right)
if left_err != nil && right_err != nil {
return LogicalAnd{left, right}
} else if left_err == nil && right_err != nil {
return LogicalAnd{BooleanConstant{left_boolean}, right}
} else if left_err != nil && right_err == nil {
return LogicalAnd{left, BooleanConstant{right_boolean}}
}
return BooleanConstant{left_boolean && right_boolean}
}
type LogicalOr struct {
left Expr
right Expr
}
func (expr LogicalOr) String() string {
return fmt.Sprintf("(| %s %s)", expr.left, expr.right)
}
func (expr LogicalOr) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_boolean, left_err := ToBoolean(left)
right_boolean, right_err := ToBoolean(right)
if left_err != nil && right_err != nil {
return LogicalOr{left, right}
} else if left_err == nil && right_err != nil {
return LogicalOr{BooleanConstant{left_boolean}, right}
} else if left_err != nil && right_err == nil {
return LogicalOr{left, BooleanConstant{right_boolean}}
}
return BooleanConstant{left_boolean || right_boolean}
}
type LogicalXor struct {
left Expr
right Expr
}
func (expr LogicalXor) String() string {
return fmt.Sprintf("(^ %s %s)", expr.left, expr.right)
}
func (expr LogicalXor) Value(symbols map[string]Expr) Expr {
left := expr.left.Value(symbols)
right := expr.right.Value(symbols)
left_boolean, left_err := ToBoolean(left)
right_boolean, right_err := ToBoolean(right)
if left_err != nil && right_err != nil {
return LogicalXor{left, right}
} else if left_err == nil && right_err != nil {
return LogicalXor{BooleanConstant{left_boolean}, right}
} else if left_err != nil && right_err == nil {
return LogicalXor{left, BooleanConstant{right_boolean}}
}
return BooleanConstant{left_boolean != right_boolean}
}
func ParseLiteralExpr(token Token) (Expr, error) {
switch token.Type {
case TOKEN_NUMERIC_LITERAL:
val, err := strconv.ParseFloat(token.Text, 64)
if err != nil {
return nil, err
}
return NumericConstant{val}, nil
case TOKEN_BOOLEAN_LITERAL:
if token.Text == "TRUE" {
return BooleanConstant{true}, nil
} else {
return BooleanConstant{false}, nil
}
default:
return nil, fmt.Errorf("Unknown literal token type: %02x", token.Type)
}
}
func ParseNestedExpr(stream *TokenStream) (Expr, error) {
level := 1
expr_tokens := []Token{}
for level > 0 {
tokens, err := stream.Read(1)
if err != nil {
return nil, err
} else if tokens[0].Type == TOKEN_LEFT_PAREN {
level += 1
} else if tokens[0].Type == TOKEN_RIGHT_PAREN {
level -= 1
}
if level != 0 {
expr_tokens = append(expr_tokens, tokens[0])
}
}
sub_stream := NewTokenStream(expr_tokens)
return ParseExpr(sub_stream)
}
var BinaryOperatorFactories = map[string]func(Expr, Expr)Expr{
"+": func(left, right Expr) Expr {
return NumericAdd{left, right}
},
"-": func(left, right Expr) Expr {
return NumericSub{left, right}
},
"*": func(left, right Expr) Expr {
return NumericMul{left, right}
},
"**": func(left, right Expr) Expr {
return NumericPow{left, right}
},
"/": func(left, right Expr) Expr {
return NumericDiv{left, right}
},
"&": func(left, right Expr) Expr {
return LogicalAnd{left, right}
},
"|": func(left, right Expr) Expr {
return LogicalOr{left, right}
},
"^": func(left, right Expr) Expr {
return LogicalXor{left, right}
},
"==": func(left, right Expr) Expr {
return NumericEQ{left, right}
},
"!=": func(left, right Expr) Expr {
return NumericNEQ{left, right}
},
">=": func(left, right Expr) Expr {
return NumericGEQ{left, right}
},
"<=": func(left, right Expr) Expr {
return NumericLEQ{left, right}
},
">": func(left, right Expr) Expr {
return NumericGT{left, right}
},
"<": func(left, right Expr) Expr {
return NumericLT{left, right}
},
}
var UnaryOperatorFactories = map[string]func(Expr)Expr{
"~": func(sub_expr Expr) Expr {
return LogicalNot{sub_expr}
},
"-": func(sub_expr Expr) Expr {
return NumericNegate{sub_expr}
},
}
func ParseUnaryExpr(stream *TokenStream, operator Token) (Expr, error) {
factory, found := UnaryOperatorFactories[operator.Text]
if found == false {
return nil, fmt.Errorf("%+v not in factory map for unary operators", operator)
}
sub_expr, err := ParseExpr(stream)
if err != nil {
return nil, err
}
return factory(sub_expr), nil
}
func ParseBinaryExpr(stream *TokenStream, left Expr, operator Token) (Expr, error) {
factory, found := BinaryOperatorFactories[operator.Text]
if found == false {
return nil, fmt.Errorf("%+v not in factory map for binary operators", operator.Text)
}
right, err := ParseExpr(stream)
if err != nil {
return nil, err
}
return factory(left, right), nil
}
func ParseExpr(stream *TokenStream) (Expr, error) {
tokens, err := stream.Read(1)
if err != nil {
return nil, err
}
var left Expr = nil
switch tokens[0].Type {
case TOKEN_UNARY_OPERATOR:
fallthrough
case TOKEN_AMBIGUOUS_OPERATOR:
left, err = ParseUnaryExpr(stream, tokens[0])
case TOKEN_LEFT_PAREN:
left, err = ParseNestedExpr(stream)
case TOKEN_SYMBOL:
left = SymbolExpr{tokens[0].Text}
case TOKEN_NUMERIC_LITERAL:
fallthrough
case TOKEN_BOOLEAN_LITERAL:
left, err = ParseLiteralExpr(tokens[0])
default:
err = fmt.Errorf("unexpected token %+v", tokens[0])
}
if err != nil {
return nil, err
}
next_tokens, err := stream.Peek(1)
if errors.Is(err, io.EOF) {
return left, nil
} else if err != nil {
return nil, err
} else if next_tokens[0].Type == TOKEN_BINARY_OPERATOR || next_tokens[0].Type == TOKEN_AMBIGUOUS_OPERATOR {
next_tokens, err := stream.Read(1)
if err != nil {
return nil, err
}
return ParseBinaryExpr(stream, left, next_tokens[0])
} else {
return left, nil
}
}

@ -0,0 +1,23 @@
package boolean
import (
"testing"
"fmt"
"bufio"
"strings"
)
func TestSimpleExpr(t *testing.T) {
stream := bufio.NewReader(strings.NewReader("(x + 10) + 5 + -10"))
token_stream, err := Tokenize(stream)
if err != nil {
t.Error(err)
}
expr, err := ParseExpr(token_stream)
if err != nil {
t.Error(err)
}
fmt.Printf("EVAL : %s - %s\n", expr, expr.Value(map[string]Expr{
"x": NumericConstant{2.5},
}))
}

@ -0,0 +1,3 @@
module git.metznet.ca/MetzNet/boolean
go 1.21.5

@ -0,0 +1,320 @@
package boolean
import (
"bufio"
"io"
"fmt"
"errors"
"slices"
"log/slog"
)
type TokenType int
type ExprType int
const (
EXPR_NUMERIC_CONSTANT ExprType = iota
EXPR_BOOLEAN_CONSTNAT
TOKEN_ERR TokenType = iota
TOKEN_UNARY_OPERATOR
TOKEN_BINARY_OPERATOR
TOKEN_AMBIGUOUS_OPERATOR
TOKEN_LEFT_PAREN
TOKEN_RIGHT_PAREN
TOKEN_SYMBOL
TOKEN_NUMERIC_LITERAL
TOKEN_BOOLEAN_LITERAL
)
type Token struct {
Text string
Type TokenType
Index uint
}
type TokenStream struct {
Tokens []Token
Position int
}
func (stream *TokenStream) Peek(n int) ([]Token, error) {
if stream.Position >= len(stream.Tokens) {
return nil, io.EOF
}
return stream.Tokens[stream.Position:stream.Position+n], nil
}
func (stream *TokenStream) Read(n int) ([]Token, error) {
if stream.Position >= len(stream.Tokens) {
return nil, io.EOF
}
ret := stream.Tokens[stream.Position:stream.Position+n]
stream.Position += len(ret)
return ret, nil
}
func NewTokenStream(tokens []Token) *TokenStream {
return &TokenStream{
Tokens: tokens,
Position: 0,
}
}
type OperatorTree struct {
TokenType
Next map[byte]OperatorTree
}
func SymbolValidStart(char byte) bool {
if char >= 97 && char <= 122 {
return true
} else if char >= 65 && char <= 90 {
return true
} else if char == '_' {
return true
}
return false
}
func SymbolValidCont(char byte) bool {
if SymbolValidStart(char) {
return true
} else if char >= 48 && char <= 57 {
return true
} else if char == '-' {
return true
}
return false
}
func NumericValid(char byte) bool {
if char >= 48 && char <= 57 {
return true
}
return false
}
var (
Whitepace = []byte{' ', '\t', '\n', '\r'}
OperatorTokens = map[byte]OperatorTree{
'.': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
'~': {
TokenType: TOKEN_UNARY_OPERATOR,
Next: nil,
},
'&': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
'|': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
'^': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
'+': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
'*': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: map[byte]OperatorTree{
'*': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
},
},
'/': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
'-': {
TokenType: TOKEN_AMBIGUOUS_OPERATOR,
Next: nil,
},
'!': {
TokenType: TOKEN_ERR,
Next: map[byte]OperatorTree{
'=': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
},
},
'=': {
TokenType: TOKEN_ERR,
Next: map[byte]OperatorTree{
'=': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
},
},
'>': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: map[byte]OperatorTree{
'=': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
},
},
'<': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: map[byte]OperatorTree{
'=': {
TokenType: TOKEN_BINARY_OPERATOR,
Next: nil,
},
},
},
}
)
func TokenizeOperator(stream *bufio.Reader, node *OperatorTree, current string) (TokenType, string, error) {
next_chars, err := stream.Peek(1)
if errors.Is(err, io.EOF) {
return node.TokenType, current, nil
} else if err != nil {
return TOKEN_ERR, current, fmt.Errorf("TokenizeOperator peek error: %w", err)
}
next_node, continues := node.Next[next_chars[0]]
if continues == true {
_, err := stream.ReadByte()
if err != nil {
return TOKEN_ERR, current, fmt.Errorf("TokenizeOperator consume error: %w", err)
}
return TokenizeOperator(stream, &next_node, current + string(next_chars))
} else {
return node.TokenType, current, nil
}
}
func Tokenize(stream *bufio.Reader) (*TokenStream, error) {
tokens := []Token{}
var position uint = 0
for true {
char, err := stream.ReadByte()
if errors.Is(err, io.EOF) {
break
} else if err != nil {
return nil, fmt.Errorf("tokenize read error: %w", err)
}
if slices.Contains(Whitepace, char) {
slog.Debug("tokenizer", "whitespace", char)
position += 1
continue
} else if node, is_operator := OperatorTokens[char]; is_operator == true{
token_type, string, err := TokenizeOperator(stream, &node, string(char))
if err != nil {
return nil, err
}
slog.Debug("tokenizer", "operator", string)
tokens = append(tokens, Token{
Type: token_type,
Text: string,
Index: position,
})
position += uint(len(string))
} else if NumericValid(char) {
literal := string(char)
decimal := false
for true {
next_chars, err := stream.Peek(1)
if errors.Is(err, io.EOF) {
break
} else if err != nil {
return nil, fmt.Errorf("numeric peek error: %w", err)
} else if NumericValid(next_chars[0]) {
_, err := stream.ReadByte()
if err != nil {
return nil, fmt.Errorf("numeric read error: %w", err)
}
literal += string(next_chars)
} else if next_chars[0] == '.' {
if decimal == true {
break
}
decimal = true
_, err := stream.ReadByte()
if err != nil {
return nil, fmt.Errorf("numeric read error: %w", err)
}
literal += string(next_chars)
} else {
break
}
}
slog.Debug("tokenizer", "numeric", literal)
tokens = append(tokens, Token{
Type: TOKEN_NUMERIC_LITERAL,
Text: literal,
Index: position,
})
position += uint(len(literal))
} else if SymbolValidStart(char) {
symbol := string(char)
for true {
next_chars, err := stream.Peek(1)
if errors.Is(err, io.EOF) {
break
} else if err != nil {
return nil, fmt.Errorf("symbol peek error: %w", err)
} else if SymbolValidCont(next_chars[0]) == true {
_, err := stream.ReadByte()
if err != nil {
return nil, fmt.Errorf("symbol read error: %w", err)
}
symbol += string(next_chars)
} else {
break
}
}
slog.Debug("tokenizer", "symbol", symbol)
token_type := TOKEN_SYMBOL
switch symbol {
case "TRUE":
token_type = TOKEN_BOOLEAN_LITERAL
case "FALSE":
token_type = TOKEN_BOOLEAN_LITERAL
}
tokens = append(tokens, Token{
Type: token_type,
Text: symbol,
Index: position,
})
position += uint(len(symbol))
} else {
switch char {
case '(':
tokens = append(tokens, Token{
Type: TOKEN_LEFT_PAREN,
Text: "(",
Index: position,
})
position += 1
case ')':
tokens = append(tokens, Token{
Type: TOKEN_RIGHT_PAREN,
Text: ")",
Index: position,
})
position += 1
default:
return nil, fmt.Errorf("tokenize unexpected character: %c", char)
}
}
}
return NewTokenStream(tokens), nil
}

@ -0,0 +1,73 @@
package boolean
import (
"testing"
"bufio"
"strings"
"fmt"
"io"
)
func TestTokenize(t *testing.T) {
stream := bufio.NewReader(strings.NewReader("10.0a + 10.0"))
token_stream, err := Tokenize(stream)
if err != nil {
t.Error(err)
} else {
for true {
tokens, err := token_stream.Read(1)
if err == io.EOF {
break
} else if err != nil {
t.Error(err)
} else {
fmt.Printf("Token: %+v\n", tokens[0])
}
}
}
}
func TestLogicalNot(t *testing.T) {
expr := LogicalNot{BooleanConstant{true}}
val := expr.Value(nil)
fmt.Printf("LogicalNot %+v, Result: %+v\n", expr, val)
}
func TestNumericNegate(t *testing.T) {
expr := NumericNegate{NumericConstant{2.4}}
val := expr.Value(nil)
fmt.Printf("NumericNegate %+v, Result: %+v\n", expr, val)
}
func TestNumericAdd(t *testing.T) {
expr := NumericAdd{SymbolExpr{"test"}, NumericConstant{1.3}}
val := expr.Value(map[string]Expr{
"test": NumericConstant{1.2},
})
fmt.Printf("NumericAdd %+v, Result: %+v\n", expr, val)
}
func TestLogicalXor(t *testing.T) {
expr := LogicalXor{BooleanConstant{false}, BooleanConstant{false}}
val := expr.Value(nil)
fmt.Printf("LogicalXor %+v, Result: %+v\n", expr, val)
}
func TestBadLiteral(t *testing.T) {
stream := bufio.NewReader(strings.NewReader("10.0.0"))
token_stream, err := Tokenize(stream)
if err != nil {
t.Error(err)
} else {
for true {
tokens, err := token_stream.Read(1)
if err == io.EOF {
break
} else if err != nil {
t.Error(err)
} else {
fmt.Printf("Token: %+v\n", tokens[0])
}
}
}
}