ekuiper-new/internal/xsql/lexical.go

// Copyright 2021-2023 EMQ Technologies Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package xsql

import (
	"bufio"
	"bytes"
	"io"
	"strconv"
	"strings"

	"github.com/lf-edge/ekuiper/pkg/ast"
)

type Scanner struct {
	r   *bufio.Reader
	buf *bytes.Buffer
}

func NewScanner(r io.Reader) *Scanner {
	return &Scanner{r: bufio.NewReader(r), buf: &bytes.Buffer{}}
}

func (s *Scanner) Scan() (tok ast.Token, lit string) {
	ch := s.read()
	if isWhiteSpace(ch) {
		// s.unread()
		return s.ScanWhiteSpace()
	} else if isLetter(ch) {
		s.unread()
		return s.ScanIdent()
	} else if isQuotation(ch) {
		s.unread()
		return s.ScanString(ch == '\'')
	} else if isDigit(ch) {
		s.unread()
		return s.ScanNumber(false, false)
	} else if isBackquote(ch) {
		return s.ScanBackquoteIdent()
	}

	switch ch {
	case eof:
		return ast.EOF, ast.Tokens[ast.EOF]
	case '=':
		return ast.EQ, ast.Tokens[ast.EQ]
	case '!':
		_, _ = s.ScanWhiteSpace()
		if r := s.read(); r == '=' {
			return ast.NEQ, ast.Tokens[ast.NEQ]
		} else {
			s.unread()
		}
		return ast.EQ, ast.Tokens[ast.EQ]
	case '<':
		_, _ = s.ScanWhiteSpace()
		if r := s.read(); r == '=' {
			return ast.LTE, ast.Tokens[ast.LTE]
		} else {
			s.unread()
		}
		return ast.LT, ast.Tokens[ast.LT]
	case '>':
		_, _ = s.ScanWhiteSpace()
		if r := s.read(); r == '=' {
			return ast.GTE, ast.Tokens[ast.GTE]
		} else {
			s.unread()
		}
		return ast.GT, ast.Tokens[ast.GT]
	case '+':
		return ast.ADD, ast.Tokens[ast.ADD]
	case '-':
		_, _ = s.ScanWhiteSpace()
		if r := s.read(); r == '-' {
			s.skipUntilNewline()
			return ast.COMMENT, ""
		} else if r == '>' {
			return ast.ARROW, ast.Tokens[ast.ARROW]
		} else if r == '.' {
			_, _ = s.ScanWhiteSpace()
			if r1 := s.read(); isDigit(r1) {
				s.unread()
				return s.ScanNumber(true, true)
			} else {
				s.unread()
			}
			s.unread()
		} else {
			s.unread()
		}
		return ast.SUB, ast.Tokens[ast.SUB]
	case '/':
		_, _ = s.ScanWhiteSpace()
		if r := s.read(); r == '*' {
			if err := s.skipUntilEndComment(); err != nil {
				return ast.ILLEGAL, ""
			}
			return ast.COMMENT, ""
		} else {
			s.unread()
		}
		return ast.DIV, ast.Tokens[ast.DIV]
	case '.':
		if r := s.read(); isDigit(r) {
			s.unread()
			return s.ScanNumber(true, false)
		}
		s.unread()
		return ast.DOT, ast.Tokens[ast.DOT]
	case '%':
		return ast.MOD, ast.Tokens[ast.MOD]
	case '&':
		return ast.BITWISE_AND, ast.Tokens[ast.BITWISE_AND]
	case '|':
		return ast.BITWISE_OR, ast.Tokens[ast.BITWISE_OR]
	case '^':
		return ast.BITWISE_XOR, ast.Tokens[ast.BITWISE_XOR]
	case '*':
		return ast.ASTERISK, ast.Tokens[ast.ASTERISK]
	case ',':
		return ast.COMMA, ast.Tokens[ast.COMMA]
	case '(':
		return ast.LPAREN, ast.Tokens[ast.LPAREN]
	case ')':
		return ast.RPAREN, ast.Tokens[ast.RPAREN]
	case '[':
		return ast.LBRACKET, ast.Tokens[ast.LBRACKET]
	case ']':
		return ast.RBRACKET, ast.Tokens[ast.RBRACKET]
	case ':':
		return ast.COLON, ast.Tokens[ast.COLON]
	case '#':
		return ast.HASH, ast.Tokens[ast.HASH]
	case ';':
		return ast.SEMICOLON, ast.Tokens[ast.SEMICOLON]
	}
	return ast.ILLEGAL, ""
}

func (s *Scanner) ScanIdent() (tok ast.Token, lit string) {
	s.buf.Reset()
	s.buf.WriteRune(s.read())
	for {
		if ch := s.read(); ch == eof {
			break
		} else if !isLetter(ch) && !isDigit(ch) && ch != '_' {
			s.unread()
			break
		} else {
			s.buf.WriteRune(ch)
		}
	}
	word := s.buf.String()
	switch lit = strings.ToUpper(word); lit {
	case "SELECT":
		return ast.SELECT, lit
	case "AS":
		return ast.AS, lit
	case "FROM":
		return ast.FROM, lit
	case "WHERE":
		return ast.WHERE, lit
	case "AND":
		return ast.AND, lit
	case "OR":
		return ast.OR, lit
	case "GROUP":
		return ast.GROUP, lit
	case "HAVING":
		return ast.HAVING, lit
	case "ORDER":
		return ast.ORDER, lit
	case "BY":
		return ast.BY, lit
	case "DESC":
		return ast.DESC, lit
	case "ASC":
		return ast.ASC, lit
	case "FILTER":
		return ast.FILTER, lit
	case "INNER":
		return ast.INNER, lit
	case "LEFT":
		return ast.LEFT, lit
	case "RIGHT":
		return ast.RIGHT, lit
	case "FULL":
		return ast.FULL, lit
	case "CROSS":
		return ast.CROSS, lit
	case "JOIN":
		return ast.JOIN, lit
	case "ON":
		return ast.ON, lit
	case "CASE":
		return ast.CASE, lit
	case "WHEN":
		return ast.WHEN, lit
	case "THEN":
		return ast.THEN, lit
	case "ELSE":
		return ast.ELSE, lit
	case "END":
		return ast.END, lit
	case "IN":
		return ast.IN, lit
	case "NOT":
		return ast.NOT, lit
	case "BETWEEN":
		return ast.BETWEEN, lit
	case "LIKE":
		return ast.LIKE, lit
	case "OVER":
		return ast.OVER, lit
	case "PARTITION":
		return ast.PARTITION, lit
	case "REPLACE":
		return ast.REPLACE, lit
	case "EXCEPT":
		return ast.EXCEPT, lit
	case "TRUE":
		return ast.TRUE, lit
	case "FALSE":
		return ast.FALSE, lit
	case "DD":
		return ast.DD, lit
	case "HH":
		return ast.HH, lit
	case "MI":
		return ast.MI, lit
	case "SS":
		return ast.SS, lit
	case "MS":
		return ast.MS, lit
	}
	return ast.IDENT, word
}

func (s *Scanner) ScanString(isSingle bool) (tok ast.Token, lit string) {
	s.buf.Reset()
	ch := s.read()
	if ch == '\'' && isSingle {
		s.buf.WriteRune('"')
	} else {
		s.buf.WriteRune(ch)
	}
	escape := false
	for {
		ch = s.read()
		if ch == '"' && !escape {
			if !isSingle {
				s.buf.WriteRune(ch)
				break
			} else {
				escape = false
				s.buf.WriteRune('\\')
				s.buf.WriteRune(ch)
			}
		} else if ch == '\'' && !escape && isSingle {
			s.buf.WriteRune('"')
			break
		} else if ch == eof {
			return ast.BADSTRING, s.buf.String()
		} else if ch == '\\' && !escape {
			escape = true
			nextCh := s.read()
			if nextCh == '\'' && isSingle {
				s.buf.WriteRune(nextCh)
			} else {
				s.buf.WriteRune(ch)
				s.unread()
			}
		} else {
			escape = false
			s.buf.WriteRune(ch)
		}
	}
	word := s.buf.String()
	r, err := strconv.Unquote(word)
	if err != nil {
		return ast.ILLEGAL, "invalid string: " + word
	}
	if isSingle {
		return ast.SINGLEQUOTE, r
	}
	return ast.STRING, r
}

func (s *Scanner) ScanDigit() (tok ast.Token, lit string) {
	s.buf.Reset()
	ch := s.read()
	s.buf.WriteRune(ch)
	for {
		if ch := s.read(); isDigit(ch) {
			s.buf.WriteRune(ch)
		} else {
			s.unread()
			break
		}
	}
	return ast.INTEGER, s.buf.String()
}

func (s *Scanner) ScanNumber(startWithDot bool, isNeg bool) (tok ast.Token, lit string) {
	s.buf.Reset()

	if isNeg {
		s.buf.WriteRune('-')
	}

	if startWithDot {
		s.buf.WriteRune('.')
	}

	ch := s.read()
	s.buf.WriteRune(ch)

	isNum := false
	for {
		if ch := s.read(); isDigit(ch) {
			s.buf.WriteRune(ch)
		} else if ch == '.' {
			isNum = true
			s.buf.WriteRune(ch)
		} else {
			s.unread()
			break
		}
	}
	if isNum || startWithDot {
		return ast.NUMBER, s.buf.String()
	} else {
		return ast.INTEGER, s.buf.String()
	}
}

func (s *Scanner) ScanBackquoteIdent() (tok ast.Token, lit string) {
	s.buf.Reset()
	for {
		ch := s.read()
		if isBackquote(ch) || ch == eof {
			break
		}
		s.buf.WriteRune(ch)
	}
	return ast.IDENT, s.buf.String()
}

func (s *Scanner) skipUntilNewline() {
	for {
		if ch := s.read(); ch == '\n' || ch == eof {
			return
		}
	}
}

func (s *Scanner) skipUntilEndComment() error {
	for {
		if ch1 := s.read(); ch1 == '*' {
			// We might be at the end.
		star:
			ch2 := s.read()
			if ch2 == '/' {
				return nil
			} else if ch2 == '*' {
				// We are back in the state machine since we see a star.
				goto star
			} else if ch2 == eof {
				return io.EOF
			}
		} else if ch1 == eof {
			return io.EOF
		}
	}
}

func (s *Scanner) ScanWhiteSpace() (tok ast.Token, lit string) {
	s.buf.Reset()
	for {
		if ch := s.read(); ch == eof {
			break
		} else if !isWhiteSpace(ch) {
			s.unread()
			break
		} else {
			s.buf.WriteRune(ch)
		}
	}
	return ast.WS, s.buf.String()
}

func (s *Scanner) read() rune {
	ch, _, err := s.r.ReadRune()
	if err != nil {
		return eof
	}
	return ch
}

func (s *Scanner) unread() {
	_ = s.r.UnreadRune()
}

var eof = rune(0)

func isWhiteSpace(r rune) bool {
	return (r == ' ') || (r == '\t') || (r == '\r') || (r == '\n')
}

func isLetter(ch rune) bool { return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z') }

func isDigit(ch rune) bool { return ch >= '0' && ch <= '9' }

func isQuotation(ch rune) bool { return ch == '"' || ch == '\'' }

func isBackquote(ch rune) bool { return ch == '`' }