Added initial commit for the tokenizer

Author: djthorpe

README.md

@@ -1,2 +1,3 @@
 # go-tokenizer
+
 General Tokenizer and Abstract Syntax Tree Generator

doc.go

@@ -0,0 +1,4 @@
+/*
+The `tokenizer` package implements a generic expression scanner for tokens
+*/
+package tokenizer

errors.go

@@ -0,0 +1,56 @@
+package tokenizer
+
+import "fmt"
+
+///////////////////////////////////////////////////////////////////////////////
+// TYPES
+
+type Error uint
+
+///////////////////////////////////////////////////////////////////////////////
+// GLOBALS
+
+const (
+	ErrSuccess Error = iota
+	ErrBadParameter
+	ErrDuplicateEntry
+	ErrUnexpectedResponse
+	ErrNotFound
+	ErrNotModified
+	ErrInternalAppError
+	ErrNotImplemented
+)
+
+///////////////////////////////////////////////////////////////////////////////
+// STRINGIFY
+
+func (e Error) Error() string {
+	switch e {
+	case ErrSuccess:
+		return "ErrSuccess"
+	case ErrBadParameter:
+		return "ErrBadParameter"
+	case ErrDuplicateEntry:
+		return "ErrDuplicateEntry"
+	case ErrUnexpectedResponse:
+		return "ErrUnexpectedResponse"
+	case ErrNotFound:
+		return "ErrNotFound"
+	case ErrNotModified:
+		return "ErrNotModified"
+	case ErrInternalAppError:
+		return "ErrInternalAppError"
+	case ErrNotImplemented:
+		return "ErrNotImplemented"
+	default:
+		return "[?? Invalid Error value]"
+	}
+}
+
+func (e Error) With(args ...any) error {
+	return fmt.Errorf("%w: %s", e, fmt.Sprint(args...))
+}
+
+func (e Error) Withf(format string, args ...any) error {
+	return fmt.Errorf("%w: %s", e, fmt.Sprintf(format, args...))
+}

go.mod

@@ -0,0 +1,3 @@
+module github.com/mutablelogic/go-tokenizer
+
+go 1.25.4

pos.go

@@ -0,0 +1,56 @@
+package tokenizer
+
+import (
+	"fmt"
+)
+
+///////////////////////////////////////////////////////////////////////////////
+// TYPES
+
+// Pos marks the position within a context, like a file path. If path is
+// nil then Pos is not used. Note that Line and Col are zero-indexed, so
+// 1 needs to be added to them when printing.
+type Pos struct {
+	Path *string
+	Line uint
+	Col  uint
+
+	// Record previous position for unread
+	x, y uint
+}
+
+// Error with a position
+type PosError struct {
+	Err error
+	Pos Pos
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// LIFECYCLE
+
+func NewPosError(err error, pos Pos) error {
+	return &PosError{Err: err, Pos: pos}
+}
+
+func NewParseError(t *Token) error {
+	return &PosError{Err: ErrBadParameter.Withf("Parse error near %q", t.toString()), Pos: t.Pos}
+}
+
+///////////////////////////////////////////////////////////////////////////////
+// STRINGIFY
+
+func (p *Pos) String() string {
+	if p.Path == nil || *p.Path == "" {
+		return fmt.Sprintf("pos<%d:%d>", p.Line+1, p.Col+1)
+	} else {
+		return fmt.Sprintf("pos<%s:%d:%d>", *p.Path, p.Line+1, p.Col+1)
+	}
+}
+
+func (e *PosError) Error() string {
+	if e.Pos.Path == nil || *e.Pos.Path == "" {
+		return fmt.Errorf("%d:%d: %w", e.Pos.Line+1, e.Pos.Col+1, e.Err).Error()
+	} else {
+		return fmt.Errorf("%s:%d:%d: %w", *e.Pos.Path, e.Pos.Line+1, e.Pos.Col+1, e.Err).Error()
+	}
+}

pos_test.go

@@ -0,0 +1,22 @@
+package tokenizer_test
+
+import (
+	"testing"
+
+	// Packages
+	tokenizer "github.com/mutablelogic/go-tokenizer"
+	assert "github.com/stretchr/testify/assert"
+)
+
+func Test_pos_001(t *testing.T) {
+	assert := assert.New(t)
+	pos := tokenizer.Pos{}
+	assert.Equal("pos<1:1>", pos.String())
+}
+
+func Test_pos_002(t *testing.T) {
+	assert := assert.New(t)
+	path := "test.hcl"
+	pos := tokenizer.Pos{Path: &path}
+	assert.Equal("pos<test.hcl:1:1>", pos.String())
+}