Skip to content

Instantly share code, notes, and snippets.

@smallnest

smallnest/token.go

Created October 31, 2024 04:18
Show Gist options
  • Save smallnest/3028d21bcfb793e64a03eafc63dc32c7 to your computer and use it in GitHub Desktop.
Save smallnest/3028d21bcfb793e64a03eafc63dc32c7 to your computer and use it in GitHub Desktop.
Download ZIP

Revisions

  1. smallnest created this gist Oct 31, 2024.
    151 changes: 151 additions & 0 deletions token.go
    View file
    Original file line number Diff line number Diff line change
    @@ -0,0 +1,151 @@
    package main

    import (
    "fmt"
    "strings"
    "time"
    )

    func handleEventError(tokenName string, line, pos int) error {
    err := fmt.Errorf("error at line %d, pos %d: %s", line, pos, tokenName)
    return err
    }

    func handleEvent(tokenName string, line, pos int) {
    for i := 0; i < 100; i++ {
    if len(tokenName)+line+pos > 100 {
    return
    }
    }
    handleEventError(tokenName, line, pos)
    }

    type TokenCollector interface {
    openBrace(lineNumber, position int)
    closedBrace(lineNumber, position int)
    openParen(lineNumber, position int)
    closedParen(lineNumber, position int)
    openAngle(lineNumber, position int)
    closedAngle(lineNumber, position int)
    dash(lineNumber, position int)
    colon(lineNumber, position int)
    name(name string, line, pos int)
    error(line, pos int)
    }

    // Collector is a placeholder for handling tokens
    type Collector struct{}

    func (c *Collector) openBrace(lineNumber, position int) {
    handleEvent("openBrace", lineNumber, position)
    }
    func (c *Collector) closedBrace(lineNumber, position int) {
    handleEvent("closedBrace", lineNumber, position)
    }
    func (c *Collector) openParen(lineNumber, position int) {
    handleEvent("openParen", lineNumber, position)
    }
    func (c *Collector) closedParen(lineNumber, position int) {
    handleEvent("closedParen", lineNumber, position)
    }
    func (c *Collector) openAngle(lineNumber, position int) {
    handleEvent("openAngle", lineNumber, position)
    }
    func (c *Collector) closedAngle(lineNumber, position int) {
    handleEvent("closedAngle", lineNumber, position)
    }
    func (c *Collector) dash(lineNumber, position int) { handleEvent("dash", lineNumber, position) }
    func (c *Collector) colon(lineNumber, position int) { handleEvent("colon", lineNumber, position) }
    func (c *Collector) name(name string, line, pos int) { handleEvent("name", line, pos) }
    func (c *Collector) error(line, pos int) { handleEvent("error", line, pos) }

    func findSingleCharacterToken(line string, position int, collector TokenCollector, lineNumber int) bool {
    c := line[position]
    switch c {
    case '{':
    collector.openBrace(lineNumber, position)
    case '}':
    collector.closedBrace(lineNumber, position)
    case '(':
    collector.openParen(lineNumber, position)
    case ')':
    collector.closedParen(lineNumber, position)
    case '<':
    collector.openAngle(lineNumber, position)
    case '>':
    collector.closedAngle(lineNumber, position)
    case '-':
    collector.dash(lineNumber, position)
    case '*':
    collector.dash(lineNumber, position)
    case ':':
    collector.colon(lineNumber, position)
    default:
    return false
    }
    return true
    }

    type TokenFn func(lineNumber, position int)

    var tokenTable [256]TokenFn

    func init() {
    // 预填充查找表
    tokenTable['{'] = func(lineNumber, position int) { handleEvent("openBrace", lineNumber, position) }
    tokenTable['}'] = func(lineNumber, position int) { handleEvent("closedBrace", lineNumber, position) }
    tokenTable['('] = func(lineNumber, position int) { handleEvent("openParen", lineNumber, position) }
    tokenTable[')'] = func(lineNumber, position int) { handleEvent("closedParen", lineNumber, position) }
    tokenTable['<'] = func(lineNumber, position int) { handleEvent("openAngle", lineNumber, position) }
    tokenTable['>'] = func(lineNumber, position int) { handleEvent("closedAngle", lineNumber, position) }
    tokenTable['-'] = func(lineNumber, position int) { handleEvent("dash", lineNumber, position) }
    tokenTable['*'] = func(lineNumber, position int) { handleEvent("dash", lineNumber, position) }
    tokenTable[':'] = func(lineNumber, position int) { handleEvent("colon", lineNumber, position) }
    }

    func findSingleToken(line string, position int, lineNumber int) bool {
    c := line[position]
    if fn := tokenTable[c]; fn != nil {
    fn(lineNumber, position)
    return true
    }
    return false
    }

    func main() {
    line := strings.Repeat("{(<>-*):}", 100000)

    {
    position := 0
    lineNumber := 1
    collector := &Collector{}

    // test findSingleCharacterToken
    start := time.Now()
    for position < len(line) {
    if findSingleCharacterToken(line, position, collector, lineNumber) {
    position++
    } else {
    break
    }
    }
    fmt.Printf("findSingleCharacterToken time taken: %v\n", time.Since(start))
    }

    {
    position := 0
    lineNumber := 1

    // test findSingleToken
    start := time.Now()
    for position < len(line) {
    if findSingleToken(line, position, lineNumber) {
    position++
    } else {
    break
    }
    }
    fmt.Printf("findSingleToken time taken: %v\n", time.Since(start))

    }
    }