Initial commit

cmd/eval/main.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+  "git.metznet.ca/MetzNet/boolean"
+  "fmt"
+  "bufio"
+  "os"
+  "strings"
+)
+
+func main() {
+  if len(os.Args) < 2 {
+    return
+  }
+  stream := bufio.NewReader(strings.NewReader(os.Args[1]))
+  token_stream, err := boolean.Tokenize(stream)
+  if err != nil {
+    panic(err)
+  }
+  expr, err := boolean.ParseExpr(token_stream)
+  if err != nil {
+    panic(err)
+  }
+  fmt.Printf("Expr: %s\n", expr)
+}

expr.go

@@ -0,0 +1,640 @@
+package boolean
+
+import (
+  "fmt"
+  "strconv"
+  "errors"
+  "io"
+  "math"
+)
+
+type Expr interface {
+  String() string
+  Value(symbols map[string]Expr) Expr
+}
+
+type NumericConstant struct {
+  value float64
+}
+func (val NumericConstant) String() string {
+  return fmt.Sprintf("%f", val.value)
+}
+func (val NumericConstant) Value(symbols map[string]Expr) Expr {
+  return val
+}
+
+type BooleanConstant struct {
+  value bool
+}
+func (val BooleanConstant) String() string {
+  return fmt.Sprintf("%t", val.value)
+}
+func (val BooleanConstant) Value(symbols map[string]Expr) Expr {
+  return val
+}
+
+func ToNumeric(expr Expr) (float64, error) {
+  numeric, ok := expr.(NumericConstant)
+  if ok == false {
+    boolean, ok := expr.(BooleanConstant)
+    if ok == false {
+      return 0.0, fmt.Errorf("expr is not constant, can not convert to numeric")
+    } else if boolean.value {
+      return 1.0, nil
+    } else {
+      return 0.0, nil
+    }
+  } else {
+    return numeric.value, nil
+  }
+}
+
+func ToBoolean(expr Expr) (bool, error) {
+  boolean, ok := expr.(BooleanConstant)
+  if ok == false {
+    numeric, ok := expr.(NumericConstant)
+    if ok == false {
+      return false, fmt.Errorf("expr is not constant, can not convert to numeric")
+    } else {
+      return numeric.value == 0.0, nil
+    }
+  } else {
+    return boolean.value, nil
+  }
+}
+
+type LogicalNot struct {
+  sub_expr Expr
+}
+func(val LogicalNot) String() string {
+  return fmt.Sprintf("(NOT %s)", val.sub_expr)
+}
+func(val LogicalNot) Value(symbols map[string]Expr) Expr {
+  sub_expr := val.sub_expr.Value(symbols)
+
+  boolean, err := ToBoolean(sub_expr)
+  if err != nil {
+    return LogicalNot{sub_expr}
+  }
+
+  if boolean {
+    return BooleanConstant{false}
+  } else {
+    return BooleanConstant{true}
+  }
+}
+
+type NumericNegate struct {
+  sub_expr Expr
+}
+func(val NumericNegate) String() string {
+  return fmt.Sprintf("(- %s)", val.sub_expr)
+}
+func(val NumericNegate) Value(symbols map[string]Expr) Expr {
+  sub_expr := val.sub_expr.Value(symbols)
+
+  numeric, err := ToNumeric(sub_expr)
+  if err != nil {
+    return NumericNegate{sub_expr}
+  }
+
+  return NumericConstant{-numeric}
+}
+
+type NumericAdd struct {
+  left Expr
+  right Expr
+}
+func (expr NumericAdd) String() string {
+  return fmt.Sprintf("(+ %s %s)", expr.left, expr.right)
+}
+func (expr NumericAdd) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericAdd{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericAdd{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericAdd{left, NumericConstant{right_numeric}}
+  }
+
+  return NumericConstant{left_numeric + right_numeric}
+}
+
+type NumericSub struct {
+  left Expr
+  right Expr
+}
+func (expr NumericSub) String() string {
+  return fmt.Sprintf("(- %s %s)", expr.left, expr.right)
+}
+func (expr NumericSub) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericSub{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericSub{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericSub{left, NumericConstant{right_numeric}}
+  }
+
+  return NumericConstant{left_numeric - right_numeric}
+}
+
+type ErrorExpr struct {
+  text string
+}
+func (expr ErrorExpr) String() string {
+  return fmt.Sprintf("(ERR %s)", expr.text)
+}
+func (expr ErrorExpr) Value(symbols map[string]Expr) Expr {
+  return expr
+}
+
+type NumericDiv struct {
+  left Expr
+  right Expr
+}
+func (expr NumericDiv) String() string {
+  return fmt.Sprintf("(/ %s %s)", expr.left, expr.right)
+}
+func (expr NumericDiv) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericDiv{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericDiv{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericDiv{left, NumericConstant{right_numeric}}
+  }
+
+  if right_numeric == 0 {
+    return ErrorExpr{"Divide by zero"}
+  }
+
+  return NumericConstant{left_numeric / right_numeric}
+}
+
+type NumericMul struct {
+  left Expr
+  right Expr
+}
+func (expr NumericMul) String() string {
+  return fmt.Sprintf("(* %s %s)", expr.left, expr.right)
+}
+func (expr NumericMul) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericMul{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericMul{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericMul{left, NumericConstant{right_numeric}}
+  }
+
+  return NumericConstant{left_numeric * right_numeric}
+}
+
+type NumericPow struct {
+  left Expr
+  right Expr
+}
+func (expr NumericPow) String() string {
+  return fmt.Sprintf("(** %s %s)", expr.left, expr.right)
+}
+func (expr NumericPow) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericPow{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericPow{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericPow{left, NumericConstant{right_numeric}}
+  }
+
+  return NumericConstant{math.Pow(left_numeric, right_numeric)}
+}
+
+type NumericEQ struct {
+  left Expr
+  right Expr
+}
+func (expr NumericEQ) String() string {
+  return fmt.Sprintf("(== %s %s)", expr.left, expr.right)
+}
+func (expr NumericEQ) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericEQ{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericEQ{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericEQ{left, NumericConstant{right_numeric}}
+  }
+
+  return BooleanConstant{left_numeric == right_numeric}
+}
+
+type NumericNEQ struct {
+  left Expr
+  right Expr
+}
+func (expr NumericNEQ) String() string {
+  return fmt.Sprintf("(!= %s %s)", expr.left, expr.right)
+}
+func (expr NumericNEQ) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericNEQ{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericNEQ{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericNEQ{left, NumericConstant{right_numeric}}
+  }
+
+  return BooleanConstant{left_numeric != right_numeric}
+}
+
+type NumericGT struct {
+  left Expr
+  right Expr
+}
+func (expr NumericGT) String() string {
+  return fmt.Sprintf("(> %s %s)", expr.left, expr.right)
+}
+func (expr NumericGT) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericGT{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericGT{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericGT{left, NumericConstant{right_numeric}}
+  }
+
+  return BooleanConstant{left_numeric > right_numeric}
+}
+
+type NumericLT struct {
+  left Expr
+  right Expr
+}
+func (expr NumericLT) String() string {
+  return fmt.Sprintf("(< %s %s)", expr.left, expr.right)
+}
+func (expr NumericLT) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericLT{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericLT{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericLT{left, NumericConstant{right_numeric}}
+  }
+
+  return BooleanConstant{left_numeric < right_numeric}
+}
+
+type NumericGEQ struct {
+  left Expr
+  right Expr
+}
+func (expr NumericGEQ) String() string {
+  return fmt.Sprintf("(>= %s %s)", expr.left, expr.right)
+}
+func (expr NumericGEQ) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericGEQ{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericGEQ{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericGEQ{left, NumericConstant{right_numeric}}
+  }
+
+  return BooleanConstant{left_numeric >= right_numeric}
+}
+
+type NumericLEQ struct {
+  left Expr
+  right Expr
+}
+func (expr NumericLEQ) String() string {
+  return fmt.Sprintf("(<= %s %s)", expr.left, expr.right)
+}
+func (expr NumericLEQ) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_numeric, left_err := ToNumeric(left)
+  right_numeric, right_err := ToNumeric(right)
+  if left_err != nil && right_err != nil {
+    return NumericLEQ{left, right}
+  } else if left_err == nil && right_err != nil {
+    return NumericLEQ{NumericConstant{left_numeric}, right}
+  } else if left_err != nil && right_err == nil {
+    return NumericLEQ{left, NumericConstant{right_numeric}}
+  }
+
+  return BooleanConstant{left_numeric <= right_numeric}
+}
+
+type SymbolExpr struct {
+  name string
+}
+func (expr SymbolExpr) String() string {
+  return expr.name
+}
+func (expr SymbolExpr) Value(symbols map[string]Expr) Expr {
+  if symbols == nil {
+    return expr
+  }
+
+  value, has_value := symbols[expr.name]
+  if has_value {
+    return value
+  } else {
+    return expr
+  }
+}
+
+type LogicalAnd struct {
+  left Expr
+  right Expr
+}
+func (expr LogicalAnd) String() string {
+  return fmt.Sprintf("(& %s %s)", expr.left, expr.right)
+}
+func (expr LogicalAnd) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_boolean, left_err := ToBoolean(left)
+  right_boolean, right_err := ToBoolean(right)
+  if left_err != nil && right_err != nil {
+    return LogicalAnd{left, right}
+  } else if left_err == nil && right_err != nil {
+    return LogicalAnd{BooleanConstant{left_boolean}, right}
+  } else if left_err != nil && right_err == nil {
+    return LogicalAnd{left, BooleanConstant{right_boolean}}
+  }
+
+  return BooleanConstant{left_boolean && right_boolean}
+}
+
+type LogicalOr struct {
+  left Expr
+  right Expr
+}
+func (expr LogicalOr) String() string {
+  return fmt.Sprintf("(| %s %s)", expr.left, expr.right)
+}
+func (expr LogicalOr) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_boolean, left_err := ToBoolean(left)
+  right_boolean, right_err := ToBoolean(right)
+  if left_err != nil && right_err != nil {
+    return LogicalOr{left, right}
+  } else if left_err == nil && right_err != nil {
+    return LogicalOr{BooleanConstant{left_boolean}, right}
+  } else if left_err != nil && right_err == nil {
+    return LogicalOr{left, BooleanConstant{right_boolean}}
+  }
+
+  return BooleanConstant{left_boolean || right_boolean}
+}
+
+type LogicalXor struct {
+  left Expr
+  right Expr
+}
+func (expr LogicalXor) String() string {
+  return fmt.Sprintf("(^ %s %s)", expr.left, expr.right)
+}
+func (expr LogicalXor) Value(symbols map[string]Expr) Expr {
+  left := expr.left.Value(symbols)
+  right := expr.right.Value(symbols)
+
+  left_boolean, left_err := ToBoolean(left)
+  right_boolean, right_err := ToBoolean(right)
+  if left_err != nil && right_err != nil {
+    return LogicalXor{left, right}
+  } else if left_err == nil && right_err != nil {
+    return LogicalXor{BooleanConstant{left_boolean}, right}
+  } else if left_err != nil && right_err == nil {
+    return LogicalXor{left, BooleanConstant{right_boolean}}
+  }
+
+  return BooleanConstant{left_boolean != right_boolean}
+}
+
+func ParseLiteralExpr(token Token) (Expr, error) {
+  switch token.Type {
+  case TOKEN_NUMERIC_LITERAL:
+    val, err := strconv.ParseFloat(token.Text, 64)
+    if err != nil {
+      return nil, err
+    }
+    return NumericConstant{val}, nil
+  case TOKEN_BOOLEAN_LITERAL:
+    if token.Text == "TRUE" {
+      return BooleanConstant{true}, nil
+    } else {
+      return BooleanConstant{false}, nil
+    }
+  default:
+    return nil, fmt.Errorf("Unknown literal token type: %02x", token.Type)
+  }
+}
+
+func ParseNestedExpr(stream *TokenStream) (Expr, error) {
+  level := 1
+  expr_tokens := []Token{}
+
+  for level > 0 {
+    tokens, err := stream.Read(1)
+    if err != nil {
+      return nil, err
+    } else if tokens[0].Type == TOKEN_LEFT_PAREN {
+      level += 1
+    } else if tokens[0].Type == TOKEN_RIGHT_PAREN {
+      level -= 1
+    }
+
+    if level != 0 {
+      expr_tokens = append(expr_tokens, tokens[0])
+    }
+  }
+
+  sub_stream := NewTokenStream(expr_tokens)
+  return ParseExpr(sub_stream)
+}
+
+
+var BinaryOperatorFactories = map[string]func(Expr, Expr)Expr{
+  "+": func(left, right Expr) Expr {
+    return NumericAdd{left, right}
+  },
+  "-": func(left, right Expr) Expr {
+    return NumericSub{left, right}
+  },
+  "*": func(left, right Expr) Expr {
+    return NumericMul{left, right}
+  },
+  "**": func(left, right Expr) Expr {
+    return NumericPow{left, right}
+  },
+  "/": func(left, right Expr) Expr {
+    return NumericDiv{left, right}
+  },
+  "&": func(left, right Expr) Expr {
+    return LogicalAnd{left, right}
+  },
+  "|": func(left, right Expr) Expr {
+    return LogicalOr{left, right}
+  },
+  "^": func(left, right Expr) Expr {
+    return LogicalXor{left, right}
+  },
+  "==": func(left, right Expr) Expr {
+    return NumericEQ{left, right}
+  },
+  "!=": func(left, right Expr) Expr {
+    return NumericNEQ{left, right}
+  },
+  ">=": func(left, right Expr) Expr {
+    return NumericGEQ{left, right}
+  },
+  "<=": func(left, right Expr) Expr {
+    return NumericLEQ{left, right}
+  },
+  ">": func(left, right Expr) Expr {
+    return NumericGT{left, right}
+  },
+  "<": func(left, right Expr) Expr {
+    return NumericLT{left, right}
+  },
+}
+
+var UnaryOperatorFactories = map[string]func(Expr)Expr{
+  "~": func(sub_expr Expr) Expr {
+    return LogicalNot{sub_expr}
+  },
+  "-": func(sub_expr Expr) Expr {
+    return NumericNegate{sub_expr}
+  },
+}
+
+func ParseUnaryExpr(stream *TokenStream, operator Token) (Expr, error) {
+  factory, found := UnaryOperatorFactories[operator.Text]
+  if found == false {
+    return nil, fmt.Errorf("%+v not in factory map for unary operators", operator)
+  }
+
+  sub_expr, err := ParseExpr(stream)
+  if err != nil {
+    return nil, err
+  }
+
+  return factory(sub_expr), nil
+}
+
+func ParseBinaryExpr(stream *TokenStream, left Expr, operator Token) (Expr, error) {
+  factory, found := BinaryOperatorFactories[operator.Text]
+  if found == false {
+    return nil, fmt.Errorf("%+v not in factory map for binary operators", operator.Text)
+  }
+
+  right, err := ParseExpr(stream)
+  if err != nil {
+    return nil, err
+  }
+
+  return factory(left, right), nil
+}
+
+func ParseExpr(stream *TokenStream) (Expr, error) {
+  tokens, err := stream.Read(1)
+  if err != nil {
+    return nil, err
+  }
+
+  var left Expr = nil
+  switch tokens[0].Type {
+  case TOKEN_UNARY_OPERATOR:
+    fallthrough
+  case TOKEN_AMBIGUOUS_OPERATOR:
+    left, err = ParseUnaryExpr(stream, tokens[0])
+  case TOKEN_LEFT_PAREN:
+    left, err = ParseNestedExpr(stream)
+  case TOKEN_SYMBOL:
+    left = SymbolExpr{tokens[0].Text}
+  case TOKEN_NUMERIC_LITERAL:
+    fallthrough
+  case TOKEN_BOOLEAN_LITERAL:
+    left, err = ParseLiteralExpr(tokens[0])
+  default:
+    err = fmt.Errorf("unexpected token %+v", tokens[0])
+  }
+
+  if err != nil {
+    return nil, err
+  }
+
+  next_tokens, err := stream.Peek(1)
+  if errors.Is(err, io.EOF) {
+    return left, nil
+  } else if err != nil {
+    return nil, err
+  } else if next_tokens[0].Type == TOKEN_BINARY_OPERATOR || next_tokens[0].Type == TOKEN_AMBIGUOUS_OPERATOR {
+    next_tokens, err := stream.Read(1)
+    if err != nil {
+      return nil, err
+    }
+    return ParseBinaryExpr(stream, left, next_tokens[0])
+  } else {
+    return left, nil
+  }
+}

expr_test.go

@@ -0,0 +1,23 @@
+package boolean
+
+import (
+  "testing"
+  "fmt"
+  "bufio"
+  "strings"
+)
+
+func TestSimpleExpr(t *testing.T) {
+  stream := bufio.NewReader(strings.NewReader("(x + 10) + 5 + -10"))
+  token_stream, err := Tokenize(stream)
+  if err != nil {
+    t.Error(err)
+  }
+  expr, err := ParseExpr(token_stream)
+  if err != nil {
+    t.Error(err)
+  }
+  fmt.Printf("EVAL : %s - %s\n", expr, expr.Value(map[string]Expr{
+    "x": NumericConstant{2.5},
+  }))
+}

go.mod

@@ -0,0 +1,3 @@
+module git.metznet.ca/MetzNet/boolean
+
+go 1.21.5

tokenizer.go

@@ -0,0 +1,320 @@
+package boolean
+
+import (
+  "bufio"
+  "io"
+  "fmt"
+  "errors"
+  "slices"
+  "log/slog"
+)
+
+type TokenType int
+type ExprType int
+
+const (
+  EXPR_NUMERIC_CONSTANT ExprType = iota
+  EXPR_BOOLEAN_CONSTNAT
+
+  TOKEN_ERR TokenType = iota
+  TOKEN_UNARY_OPERATOR
+  TOKEN_BINARY_OPERATOR
+  TOKEN_AMBIGUOUS_OPERATOR
+  TOKEN_LEFT_PAREN
+  TOKEN_RIGHT_PAREN
+  TOKEN_SYMBOL
+  TOKEN_NUMERIC_LITERAL
+  TOKEN_BOOLEAN_LITERAL
+)
+
+type Token struct {
+  Text string
+  Type TokenType
+  Index uint
+}
+
+type TokenStream struct {
+  Tokens []Token
+  Position int
+}
+func (stream *TokenStream) Peek(n int) ([]Token, error) {
+  if stream.Position >= len(stream.Tokens) {
+    return nil, io.EOF
+  }
+  return stream.Tokens[stream.Position:stream.Position+n], nil
+}
+func (stream *TokenStream) Read(n int) ([]Token, error) {
+  if stream.Position >= len(stream.Tokens) {
+    return nil, io.EOF
+  }
+  ret := stream.Tokens[stream.Position:stream.Position+n]
+  stream.Position += len(ret)
+  return ret, nil
+}
+
+func NewTokenStream(tokens []Token) *TokenStream {
+  return &TokenStream{
+    Tokens: tokens,
+    Position: 0,
+  }
+}
+
+type OperatorTree struct {
+  TokenType
+  Next map[byte]OperatorTree
+}
+
+func SymbolValidStart(char byte) bool {
+  if char >= 97 && char <= 122 {
+    return true
+  } else if char >= 65 && char <= 90 {
+    return true
+  } else if char == '_' {
+    return true
+  }
+  return false
+}
+
+func SymbolValidCont(char byte) bool {
+  if SymbolValidStart(char) {
+    return true
+  } else if char >= 48 && char <= 57 {
+    return true
+  } else if char == '-' {
+    return true
+  }
+  return false
+}
+
+func NumericValid(char byte) bool {
+  if char >= 48 && char <= 57 {
+    return true
+  }
+  return false
+}
+
+var (
+  Whitepace = []byte{' ', '\t', '\n', '\r'}
+  OperatorTokens = map[byte]OperatorTree{
+    '.': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: nil,
+    },
+    '~': {
+      TokenType: TOKEN_UNARY_OPERATOR,
+      Next: nil,
+    },
+    '&': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: nil,
+    },
+    '|': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: nil,
+    },
+    '^': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: nil,
+    },
+    '+': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: nil,
+    },
+    '*': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: map[byte]OperatorTree{
+        '*': {
+          TokenType: TOKEN_BINARY_OPERATOR,
+          Next: nil,
+        },
+      },
+    },
+    '/': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: nil,
+    },
+    '-': {
+      TokenType: TOKEN_AMBIGUOUS_OPERATOR,
+      Next: nil,
+    },
+    '!': {
+      TokenType: TOKEN_ERR,
+      Next: map[byte]OperatorTree{
+        '=': {
+          TokenType: TOKEN_BINARY_OPERATOR,
+          Next: nil,
+        },
+      },
+    },
+    '=': {
+      TokenType: TOKEN_ERR,
+      Next: map[byte]OperatorTree{
+        '=': {
+          TokenType: TOKEN_BINARY_OPERATOR,
+          Next: nil,
+        },
+      },
+    },
+    '>': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: map[byte]OperatorTree{
+        '=': {
+          TokenType: TOKEN_BINARY_OPERATOR,
+          Next: nil,
+        },
+      },
+    },
+    '<': {
+      TokenType: TOKEN_BINARY_OPERATOR,
+      Next: map[byte]OperatorTree{
+        '=': {
+          TokenType: TOKEN_BINARY_OPERATOR,
+          Next: nil,
+        },
+      },
+    },
+  }
+)
+
+func TokenizeOperator(stream *bufio.Reader, node *OperatorTree, current string) (TokenType, string, error) {
+  next_chars, err := stream.Peek(1)
+  if errors.Is(err, io.EOF) {
+    return node.TokenType, current, nil
+  } else if err != nil {
+    return TOKEN_ERR, current, fmt.Errorf("TokenizeOperator peek error: %w", err)
+  }
+
+  next_node, continues := node.Next[next_chars[0]]
+  if continues == true {
+    _, err := stream.ReadByte()
+    if err != nil {
+      return TOKEN_ERR, current, fmt.Errorf("TokenizeOperator consume error: %w", err)
+    }
+    return TokenizeOperator(stream, &next_node, current + string(next_chars))
+  } else {
+    return node.TokenType, current, nil
+  }
+}
+
+func Tokenize(stream *bufio.Reader) (*TokenStream, error) {
+  tokens := []Token{}
+  var position uint = 0
+
+  for true {
+    char, err := stream.ReadByte()
+    if errors.Is(err, io.EOF) {
+      break
+    } else if err != nil {
+      return nil, fmt.Errorf("tokenize read error: %w", err)
+    }
+
+    if slices.Contains(Whitepace, char) {
+      slog.Debug("tokenizer", "whitespace", char)
+      position += 1
+      continue
+    } else if node, is_operator := OperatorTokens[char]; is_operator == true{
+      token_type, string, err := TokenizeOperator(stream, &node, string(char))
+      if err != nil {
+        return nil, err
+      }
+      slog.Debug("tokenizer", "operator", string)
+      tokens = append(tokens, Token{
+        Type: token_type,
+        Text: string,
+        Index: position,
+      })
+      position += uint(len(string))
+    } else if NumericValid(char) {
+      literal := string(char)
+      decimal := false
+
+      for true {
+        next_chars, err := stream.Peek(1)
+        if errors.Is(err, io.EOF) {
+          break
+        } else if err != nil {
+          return nil, fmt.Errorf("numeric peek error: %w", err)
+        } else if NumericValid(next_chars[0]) {
+          _, err := stream.ReadByte()
+          if err != nil {
+            return nil, fmt.Errorf("numeric read error: %w", err)
+          }
+          literal += string(next_chars)
+        } else if next_chars[0] == '.' {
+          if decimal == true {
+            break
+          }
+          decimal = true
+          _, err := stream.ReadByte()
+          if err != nil {
+            return nil, fmt.Errorf("numeric read error: %w", err)
+          }
+          literal += string(next_chars)
+        } else {
+          break
+        }
+      }
+      slog.Debug("tokenizer", "numeric", literal)
+      tokens = append(tokens, Token{
+        Type: TOKEN_NUMERIC_LITERAL,
+        Text: literal,
+        Index: position,
+      })
+      position += uint(len(literal))
+    } else if SymbolValidStart(char) {
+      symbol := string(char)
+      for true {
+        next_chars, err := stream.Peek(1)
+        if errors.Is(err, io.EOF) {
+          break
+        } else if err != nil {
+          return nil, fmt.Errorf("symbol peek error: %w", err)
+        } else if SymbolValidCont(next_chars[0]) == true {
+          _, err := stream.ReadByte()
+          if err != nil {
+            return nil, fmt.Errorf("symbol read error: %w", err)
+          }
+          symbol += string(next_chars)
+        } else {
+          break
+        }
+      }
+      slog.Debug("tokenizer", "symbol", symbol)
+      token_type := TOKEN_SYMBOL
+      switch symbol {
+      case "TRUE":
+        token_type = TOKEN_BOOLEAN_LITERAL
+      case "FALSE":
+        token_type = TOKEN_BOOLEAN_LITERAL
+      }
+      tokens = append(tokens, Token{
+        Type: token_type,
+        Text: symbol,
+        Index: position,
+      })
+      position += uint(len(symbol))
+    } else {
+      switch char {
+      case '(':
+        tokens = append(tokens, Token{
+          Type: TOKEN_LEFT_PAREN,
+          Text: "(",
+          Index: position,
+        })
+        position += 1
+      case ')':
+        tokens = append(tokens, Token{
+          Type: TOKEN_RIGHT_PAREN,
+          Text: ")",
+          Index: position,
+        })
+        position += 1
+      default:
+        return nil, fmt.Errorf("tokenize unexpected character: %c", char)
+      }
+    }
+  }
+
+  return NewTokenStream(tokens), nil
+}
+

tokenizer_test.go

@@ -0,0 +1,73 @@
+package boolean
+
+import (
+  "testing"
+  "bufio"
+  "strings"
+  "fmt"
+  "io"
+)
+
+func TestTokenize(t *testing.T) {
+  stream := bufio.NewReader(strings.NewReader("10.0a + 10.0"))
+  token_stream, err := Tokenize(stream)
+  if err != nil {
+    t.Error(err)
+  } else {
+    for true {
+      tokens, err := token_stream.Read(1)
+      if err == io.EOF {
+        break
+      } else if err != nil {
+        t.Error(err)
+      } else {
+        fmt.Printf("Token: %+v\n", tokens[0])
+      }
+    }
+  }
+}
+
+func TestLogicalNot(t *testing.T) {
+  expr := LogicalNot{BooleanConstant{true}}
+  val := expr.Value(nil)
+  fmt.Printf("LogicalNot %+v, Result: %+v\n", expr, val)
+}
+
+func TestNumericNegate(t *testing.T) {
+  expr := NumericNegate{NumericConstant{2.4}}
+  val := expr.Value(nil)
+  fmt.Printf("NumericNegate %+v, Result: %+v\n", expr, val)
+}
+
+func TestNumericAdd(t *testing.T) {
+  expr := NumericAdd{SymbolExpr{"test"}, NumericConstant{1.3}}
+  val := expr.Value(map[string]Expr{
+    "test": NumericConstant{1.2},
+  })
+  fmt.Printf("NumericAdd %+v, Result: %+v\n", expr, val)
+}
+
+func TestLogicalXor(t *testing.T) {
+  expr := LogicalXor{BooleanConstant{false}, BooleanConstant{false}}
+  val := expr.Value(nil)
+  fmt.Printf("LogicalXor %+v, Result: %+v\n", expr, val)
+}
+
+func TestBadLiteral(t *testing.T) {
+  stream := bufio.NewReader(strings.NewReader("10.0.0"))
+  token_stream, err := Tokenize(stream)
+  if err != nil {
+    t.Error(err)
+  } else {
+    for true {
+      tokens, err := token_stream.Read(1)
+      if err == io.EOF {
+        break
+      } else if err != nil {
+        t.Error(err)
+      } else {
+        fmt.Printf("Token: %+v\n", tokens[0])
+      }
+    }
+  }
+}